@rpascene/core 0.30.8

package/dist/lib/ai-model/action-executor.js

@@ -0,0 +1,163 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    Executor: ()=>Executor
+});
+const utils_namespaceObject = require("@rpascene/shared/utils");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class Executor {
+    markTaskAsPending(task) {
+        return {
+            status: 'pending',
+            ...task
+        };
+    }
+    async append(task) {
+        var _this_latestErrorTask, _this_latestErrorTask1;
+        (0, utils_namespaceObject.assert)('error' !== this.status, `executor is in error state, cannot append task\nerror=${null == (_this_latestErrorTask = this.latestErrorTask()) ? void 0 : _this_latestErrorTask.error}\n${null == (_this_latestErrorTask1 = this.latestErrorTask()) ? void 0 : _this_latestErrorTask1.errorStack}`);
+        if (Array.isArray(task)) this.tasks.push(...task.map((item)=>this.markTaskAsPending(item)));
+        else this.tasks.push(this.markTaskAsPending(task));
+        if ('running' !== this.status) this.status = 'pending';
+    }
+    async flush() {
+        if ('init' === this.status && this.tasks.length > 0) console.warn('illegal state for executor, status is init but tasks are not empty');
+        (0, utils_namespaceObject.assert)('running' !== this.status, 'executor is already running');
+        (0, utils_namespaceObject.assert)('completed' !== this.status, 'executor is already completed');
+        (0, utils_namespaceObject.assert)('error' !== this.status, 'executor is in error state');
+        const nextPendingIndex = this.tasks.findIndex((task)=>'pending' === task.status);
+        if (nextPendingIndex < 0) return;
+        this.status = 'running';
+        let taskIndex = nextPendingIndex;
+        let successfullyCompleted = true;
+        let previousFindOutput;
+        while(taskIndex < this.tasks.length){
+            const task = this.tasks[taskIndex];
+            (0, utils_namespaceObject.assert)('pending' === task.status, `task status should be pending, but got: ${task.status}`);
+            task.timing = {
+                start: Date.now()
+            };
+            try {
+                task.status = 'running';
+                try {
+                    if (this.onTaskStart) await this.onTaskStart(task);
+                } catch (e) {
+                    console.error('error in onTaskStart', e);
+                }
+                (0, utils_namespaceObject.assert)([
+                    'Insight',
+                    'Action',
+                    'Planning'
+                ].indexOf(task.type) >= 0, `unsupported task type: ${task.type}`);
+                const { executor, param } = task;
+                (0, utils_namespaceObject.assert)(executor, `executor is required for task type: ${task.type}`);
+                let returnValue;
+                const executorContext = {
+                    task,
+                    element: null == previousFindOutput ? void 0 : previousFindOutput.element
+                };
+                if ('Insight' === task.type) {
+                    (0, utils_namespaceObject.assert)('Locate' === task.subType || 'Query' === task.subType || 'Assert' === task.subType || 'WaitFor' === task.subType || 'Boolean' === task.subType || 'Number' === task.subType || 'String' === task.subType, `unsupported insight subType: ${task.subType}`);
+                    returnValue = await task.executor(param, executorContext);
+                    if ('Locate' === task.subType) previousFindOutput = null == returnValue ? void 0 : returnValue.output;
+                } else if ('Action' === task.type || 'Planning' === task.type) returnValue = await task.executor(param, executorContext);
+                else {
+                    console.warn(`unsupported task type: ${task.type}, will try to execute it directly`);
+                    returnValue = await task.executor(param, executorContext);
+                }
+                Object.assign(task, returnValue);
+                task.status = 'finished';
+                task.timing.end = Date.now();
+                task.timing.cost = task.timing.end - task.timing.start;
+                taskIndex++;
+            } catch (e) {
+                successfullyCompleted = false;
+                task.error = e;
+                task.errorMessage = (null == e ? void 0 : e.message) || ('string' == typeof e ? e : 'error-without-message');
+                task.errorStack = e.stack;
+                task.status = 'failed';
+                task.timing.end = Date.now();
+                task.timing.cost = task.timing.end - task.timing.start;
+                break;
+            }
+        }
+        for(let i = taskIndex + 1; i < this.tasks.length; i++)this.tasks[i].status = 'cancelled';
+        if (successfullyCompleted) this.status = 'completed';
+        else this.status = 'error';
+        if (this.tasks.length) {
+            const outputIndex = Math.min(taskIndex, this.tasks.length - 1);
+            const { thought, output } = this.tasks[outputIndex];
+            return {
+                thought,
+                output
+            };
+        }
+    }
+    isInErrorState() {
+        return 'error' === this.status;
+    }
+    latestErrorTask() {
+        if ('error' !== this.status) return null;
+        const errorTaskIndex = this.tasks.findIndex((task)=>'failed' === task.status);
+        if (errorTaskIndex >= 0) return this.tasks[errorTaskIndex];
+        return null;
+    }
+    dump() {
+        const dumpData = {
+            logTime: Date.now(),
+            name: this.name,
+            tasks: this.tasks
+        };
+        return dumpData;
+    }
+    constructor(name, options){
+        _define_property(this, "name", void 0);
+        _define_property(this, "tasks", void 0);
+        _define_property(this, "status", void 0);
+        _define_property(this, "onTaskStart", void 0);
+        this.status = (null == options ? void 0 : options.tasks) && options.tasks.length > 0 ? 'pending' : 'init';
+        this.name = name;
+        this.tasks = ((null == options ? void 0 : options.tasks) || []).map((item)=>this.markTaskAsPending(item));
+        this.onTaskStart = null == options ? void 0 : options.onTaskStart;
+    }
+}
+exports.Executor = __webpack_exports__.Executor;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "Executor"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=action-executor.js.map

package/dist/lib/ai-model/action-executor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model\\action-executor.js","sources":["webpack://@rpascene/core/webpack/runtime/define_property_getters","webpack://@rpascene/core/webpack/runtime/has_own_property","webpack://@rpascene/core/webpack/runtime/make_namespace_object","webpack://@rpascene/core/./src/ai-model/action-executor.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { assert } from '@rpascene/shared/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n  // 执行任务\n  async flush(): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n            task.subType === 'Query' ||\n            task.subType === 'Assert' ||\n            task.subType === 'WaitFor' ||\n            task.subType === 'Boolean' ||\n            task.subType === 'Number' ||\n            task.subType === 'String',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","Executor","task","_this_latestErrorTask","_this_latestErrorTask1","assert","Array","item","console","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","Date","e","executor","param","returnValue","executorContext","i","outputIndex","Math","thought","output","errorTaskIndex","dumpData","name","options"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;ACKO,MAAMI;IAyBH,kBAAkBC,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQ;YACR,GAAGA,IAAI;QACT;IACF;IAEA,MAAM,OAAOA,IAA+C,EAAiB;YAGhBC,uBAAkCC;QAF7FC,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACX,CAAC,sDAAsD,EAAE,QAAAF,CAAAA,wBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,sBAAwB,KAAK,CAAC,EAAE,EAAE,QAAAC,CAAAA,yBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,uBAAwB,UAAU,EAAE;QAEjI,IAAIE,MAAM,OAAO,CAACJ,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACK,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACL;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;IAElB;IAEA,MAAM,QAAgE;QACpE,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDM,QAAQ,IAAI,CACV;QAIJH,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QACpCA,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM,EAAc;QAEhC,MAAMI,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACP,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIO,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMR,OAAO,IAAI,CAAC,KAAK,CAACQ,UAAU;YAClCL,IAAAA,sBAAAA,MAAAA,AAAAA,EACEH,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAOW,KAAK,GAAG;YACjB;YACA,IAAI;gBACFX,KAAK,MAAM,GAAG;gBACd,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOY,GAAG;oBACVN,QAAQ,KAAK,CAAC,wBAAwBM;gBACxC;gBACAT,IAAAA,sBAAAA,MAAAA,AAAAA,EACE;oBAAC;oBAAW;oBAAU;iBAAW,CAAC,OAAO,CAACH,KAAK,IAAI,KAAK,GACxD,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEa,QAAQ,EAAEC,KAAK,EAAE,GAAGd;gBAC5BG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOU,UAAU,CAAC,oCAAoC,EAAEb,KAAK,IAAI,EAAE;gBAEnE,IAAIe;gBACJ,MAAMC,kBAAmC;oBACvChB;oBACA,SAASU,QAAAA,qBAAAA,KAAAA,IAAAA,mBAAoB,OAAO;gBACtC;gBAEA,IAAIV,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEH,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,YAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACZ,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDe,cAAc,MAAMf,KAAK,QAAQ,CAACc,OAAOE;oBACzC,IAAIhB,AAAiB,aAAjBA,KAAK,OAAO,EACdU,qBACEK,QAAAA,cAAAA,KAAAA,IAAAA,YACC,MAAM;gBAEb,OAAO,IAAIf,AAAc,aAAdA,KAAK,IAAI,IAAiBA,AAAc,eAAdA,KAAK,IAAI,EAC5Ce,cAAc,MAAMf,KAAK,QAAQ,CAACc,OAAOE;qBACpC;oBACLV,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEN,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEe,cAAc,MAAMf,KAAK,QAAQ,CAACc,OAAOE;gBAC3C;gBAEArB,OAAO,MAAM,CAACK,MAAMe;gBACpBf,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGW,KAAK,GAAG;gBAC1BX,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtDQ;YACF,EAAE,OAAOI,GAAQ;gBACfH,wBAAwB;gBACxBT,KAAK,KAAK,GAAGY;gBACbZ,KAAK,YAAY,GACfY,AAAAA,CAAAA,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO,AAAD,KAAM,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEZ,KAAK,UAAU,GAAGY,EAAE,KAAK;gBAEzBZ,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGW,KAAK,GAAG;gBAC1BX,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD;YACF;QACF;QAGA,IAAK,IAAIiB,IAAIT,YAAY,GAAGS,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAGzB,IAAIR,uBACF,IAAI,CAAC,MAAM,GAAG;aAEd,IAAI,CAAC,MAAM,GAAG;QAGhB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMS,cAAcC,KAAK,GAAG,CAACX,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEY,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAET,MAAMC,iBAAiB,IAAI,CAAC,KAAK,CAAC,SAAS,CACzC,CAACtB,OAASA,AAAgB,aAAhBA,KAAK,MAAM;QAEvB,IAAIsB,kBAAkB,GACpB,OAAO,IAAI,CAAC,KAAK,CAACA,eAAe;QAEnC,OAAO;IACT;IAEA,OAAsB;QACpB,MAAMC,WAA0B;YAC9B,SAASZ,KAAK,GAAG;YACjB,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;QACA,OAAOY;IACT;IAzLA,YACEC,IAAY,EACZC,OAEC,CACD;QAdF;QAEA;QAGA;QAEA;QAQE,IAAI,CAAC,MAAM,GACTA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAKA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAGD;QACZ,IAAI,CAAC,KAAK,GAAIC,AAAAA,CAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAK,EAAC,EAAG,GAAG,CAAC,CAACpB,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGoB,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,WAAW;IACzC;AA6KF"}

package/dist/lib/ai-model/common.js

@@ -0,0 +1,461 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    SizeSchema: ()=>SizeSchema,
+    TMultimodalPromptSchema: ()=>TMultimodalPromptSchema,
+    expandSearchArea: ()=>expandSearchArea,
+    AIActionType: ()=>common_AIActionType,
+    adaptBbox: ()=>adaptBbox,
+    TUserPromptSchema: ()=>TUserPromptSchema,
+    loadActionParam: ()=>loadActionParam,
+    fillBboxParam: ()=>fillBboxParam,
+    adaptGeminiBbox: ()=>adaptGeminiBbox,
+    getRpasceneLocationSchema: ()=>getRpasceneLocationSchema,
+    ifRpasceneLocatorField: ()=>ifRpasceneLocatorField,
+    mergeRects: ()=>mergeRects,
+    RectSchema: ()=>RectSchema,
+    PointSchema: ()=>PointSchema,
+    adaptBboxToRect: ()=>adaptBboxToRect,
+    markupImageForLLM: ()=>markupImageForLLM,
+    normalized01000: ()=>normalized01000,
+    adaptDoubaoBbox: ()=>adaptDoubaoBbox,
+    parseActionParam: ()=>parseActionParam,
+    adaptQwenBbox: ()=>adaptQwenBbox,
+    buildYamlFlowFromPlans: ()=>buildYamlFlowFromPlans,
+    dumpRpasceneLocatorField: ()=>dumpRpasceneLocatorField,
+    findAllRpasceneLocatorField: ()=>findAllRpasceneLocatorField,
+    dumpActionParam: ()=>dumpActionParam,
+    warnGPT4oSizeLimit: ()=>warnGPT4oSizeLimit
+});
+const utils_namespaceObject = require("@rpascene/shared/utils");
+const constants_namespaceObject = require("@rpascene/shared/constants");
+const extractor_namespaceObject = require("@rpascene/shared/extractor");
+const img_namespaceObject = require("@rpascene/shared/img");
+const logger_namespaceObject = require("@rpascene/shared/logger");
+const external_zod_namespaceObject = require("zod");
+var common_AIActionType = /*#__PURE__*/ function(AIActionType) {
+    AIActionType[AIActionType["ASSERT"] = 0] = "ASSERT";
+    AIActionType[AIActionType["INSPECT_ELEMENT"] = 1] = "INSPECT_ELEMENT";
+    AIActionType[AIActionType["EXTRACT_DATA"] = 2] = "EXTRACT_DATA";
+    AIActionType[AIActionType["PLAN"] = 3] = "PLAN";
+    AIActionType[AIActionType["DESCRIBE_ELEMENT"] = 4] = "DESCRIBE_ELEMENT";
+    AIActionType[AIActionType["TEXT"] = 5] = "TEXT";
+    return AIActionType;
+}({});
+const defaultBboxSize = 20;
+const debugInspectUtils = (0, logger_namespaceObject.getDebug)('ai:common');
+function fillBboxParam(locate, width, height, rightLimit, bottomLimit, vlMode) {
+    if (locate.bbox_2d && !(null == locate ? void 0 : locate.bbox)) {
+        locate.bbox = locate.bbox_2d;
+        delete locate.bbox_2d;
+    }
+    if (null == locate ? void 0 : locate.bbox) locate.bbox = adaptBbox(locate.bbox, width, height, rightLimit, bottomLimit, vlMode);
+    return locate;
+}
+function adaptQwenBbox(bbox) {
+    if (bbox.length < 2) {
+        const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;
+        throw new Error(msg);
+    }
+    const result = [
+        Math.round(bbox[0]),
+        Math.round(bbox[1]),
+        'number' == typeof bbox[2] ? Math.round(bbox[2]) : Math.round(bbox[0] + defaultBboxSize),
+        'number' == typeof bbox[3] ? Math.round(bbox[3]) : Math.round(bbox[1] + defaultBboxSize)
+    ];
+    return result;
+}
+function adaptDoubaoBbox(bbox, width, height) {
+    (0, utils_namespaceObject.assert)(width > 0 && height > 0, 'width and height must be greater than 0 in doubao mode');
+    if ('string' == typeof bbox) {
+        (0, utils_namespaceObject.assert)(/^(\d+)\s(\d+)\s(\d+)\s(\d+)$/.test(bbox.trim()), `invalid bbox data string for doubao-vision mode: ${bbox}`);
+        const splitted = bbox.split(' ');
+        if (4 === splitted.length) return [
+            Math.round(Number(splitted[0]) * width / 1000),
+            Math.round(Number(splitted[1]) * height / 1000),
+            Math.round(Number(splitted[2]) * width / 1000),
+            Math.round(Number(splitted[3]) * height / 1000)
+        ];
+        throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);
+    }
+    if (Array.isArray(bbox) && Array.isArray(bbox[0])) bbox = bbox[0];
+    let bboxList = [];
+    if (Array.isArray(bbox) && 'string' == typeof bbox[0]) bbox.forEach((item)=>{
+        if ('string' == typeof item && item.includes(',')) {
+            const [x, y] = item.split(',');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else if ('string' == typeof item && item.includes(' ')) {
+            const [x, y] = item.split(' ');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else bboxList.push(Number(item));
+    });
+    else bboxList = bbox;
+    if (4 === bboxList.length || 5 === bboxList.length) return [
+        Math.round(bboxList[0] * width / 1000),
+        Math.round(bboxList[1] * height / 1000),
+        Math.round(bboxList[2] * width / 1000),
+        Math.round(bboxList[3] * height / 1000)
+    ];
+    if (6 === bboxList.length || 2 === bboxList.length || 3 === bboxList.length || 7 === bboxList.length) return [
+        Math.max(0, Math.round(bboxList[0] * width / 1000) - defaultBboxSize / 2),
+        Math.max(0, Math.round(bboxList[1] * height / 1000) - defaultBboxSize / 2),
+        Math.min(width, Math.round(bboxList[0] * width / 1000) + defaultBboxSize / 2),
+        Math.min(height, Math.round(bboxList[1] * height / 1000) + defaultBboxSize / 2)
+    ];
+    if (8 === bbox.length) return [
+        Math.round(bboxList[0] * width / 1000),
+        Math.round(bboxList[1] * height / 1000),
+        Math.round(bboxList[4] * width / 1000),
+        Math.round(bboxList[5] * height / 1000)
+    ];
+    const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;
+    throw new Error(msg);
+}
+function adaptBbox(bbox, width, height, rightLimit, bottomLimit, vlMode) {
+    let result = [
+        0,
+        0,
+        0,
+        0
+    ];
+    result = 'doubao-vision' === vlMode || 'vlm-ui-tars' === vlMode ? adaptDoubaoBbox(bbox, width, height) : 'gemini' === vlMode ? adaptGeminiBbox(bbox, width, height) : 'qwen3-vl' === vlMode ? normalized01000(bbox, width, height) : adaptQwenBbox(bbox);
+    result[2] = Math.min(result[2], rightLimit);
+    result[3] = Math.min(result[3], bottomLimit);
+    return result;
+}
+function normalized01000(bbox, width, height) {
+    return [
+        Math.round(bbox[0] * width / 1000),
+        Math.round(bbox[1] * height / 1000),
+        Math.round(bbox[2] * width / 1000),
+        Math.round(bbox[3] * height / 1000)
+    ];
+}
+function adaptGeminiBbox(bbox, width, height) {
+    const left = Math.round(bbox[1] * width / 1000);
+    const top = Math.round(bbox[0] * height / 1000);
+    const right = Math.round(bbox[3] * width / 1000);
+    const bottom = Math.round(bbox[2] * height / 1000);
+    return [
+        left,
+        top,
+        right,
+        bottom
+    ];
+}
+function adaptBboxToRect(bbox, width, height, offsetX = 0, offsetY = 0, rightLimit = width, bottomLimit = height, vlMode) {
+    debugInspectUtils('adaptBboxToRect', bbox, width, height, 'offset', offsetX, offsetY, 'limit', rightLimit, bottomLimit, 'vlMode', vlMode);
+    const [left, top, right, bottom] = adaptBbox(bbox, width, height, rightLimit, bottomLimit, vlMode);
+    const rectLeft = left;
+    const rectTop = top;
+    let rectWidth = right - left;
+    let rectHeight = bottom - top;
+    if (rectLeft + rectWidth > width) rectWidth = width - rectLeft;
+    if (rectTop + rectHeight > height) rectHeight = height - rectTop;
+    rectWidth = Math.max(1, rectWidth);
+    rectHeight = Math.max(1, rectHeight);
+    const rect = {
+        left: rectLeft + offsetX,
+        top: rectTop + offsetY,
+        width: rectWidth,
+        height: rectHeight
+    };
+    debugInspectUtils('adaptBboxToRect, result=', rect);
+    return rect;
+}
+let warned = false;
+function warnGPT4oSizeLimit(size, modelName) {
+    if (warned) return;
+    if (modelName.toLowerCase().includes('gpt-4o')) {
+        const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your interface to a smaller resolution. Otherwise, the result may be inaccurate.`;
+        if (Math.max(size.width, size.height) > 2000 || Math.min(size.width, size.height) > 768) {
+            console.warn(warningMsg);
+            warned = true;
+        }
+    } else if (size.width > 1800 || size.height > 1800) {
+        console.warn(`The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`);
+        warned = true;
+    }
+}
+function mergeRects(rects) {
+    const minLeft = Math.min(...rects.map((r)=>r.left));
+    const minTop = Math.min(...rects.map((r)=>r.top));
+    const maxRight = Math.max(...rects.map((r)=>r.left + r.width));
+    const maxBottom = Math.max(...rects.map((r)=>r.top + r.height));
+    return {
+        left: minLeft,
+        top: minTop,
+        width: maxRight - minLeft,
+        height: maxBottom - minTop
+    };
+}
+function expandSearchArea(rect, screenSize, vlMode) {
+    const minEdgeSize = 'doubao-vision' === vlMode ? 500 : 300;
+    const defaultPadding = 160;
+    const paddingSizeHorizontal = rect.width < minEdgeSize ? Math.ceil((minEdgeSize - rect.width) / 2) : defaultPadding;
+    const paddingSizeVertical = rect.height < minEdgeSize ? Math.ceil((minEdgeSize - rect.height) / 2) : defaultPadding;
+    let newWidth = Math.max(minEdgeSize, rect.width + 2 * paddingSizeHorizontal);
+    let newHeight = Math.max(minEdgeSize, rect.height + 2 * paddingSizeVertical);
+    let newLeft = rect.left - paddingSizeHorizontal;
+    let newTop = rect.top - paddingSizeVertical;
+    if (newLeft + newWidth > screenSize.width) newLeft = screenSize.width - newWidth;
+    if (newTop + newHeight > screenSize.height) newTop = screenSize.height - newHeight;
+    newLeft = Math.max(0, newLeft);
+    newTop = Math.max(0, newTop);
+    if (newLeft + newWidth > screenSize.width) newWidth = screenSize.width - newLeft;
+    if (newTop + newHeight > screenSize.height) newHeight = screenSize.height - newTop;
+    rect.left = newLeft;
+    rect.top = newTop;
+    rect.width = newWidth;
+    rect.height = newHeight;
+    return rect;
+}
+async function markupImageForLLM(screenshotBase64, tree, size) {
+    const elementsInfo = (0, extractor_namespaceObject.treeToList)(tree);
+    const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo)=>{
+        if (elementInfo.attributes.nodeType === constants_namespaceObject.NodeType.TEXT) return false;
+        return true;
+    });
+    const imagePayload = await (0, img_namespaceObject.compositeElementInfoImg)({
+        inputImgBase64: screenshotBase64,
+        elementsPositionInfo: elementsPositionInfoWithoutText,
+        size
+    });
+    return imagePayload;
+}
+function buildYamlFlowFromPlans(plans, actionSpace, sleep) {
+    const flow = [];
+    for (const plan of plans){
+        const verb = plan.type;
+        const action = actionSpace.find((action)=>action.name === verb);
+        if (!action) {
+            console.warn(`Cannot convert action ${verb} to yaml flow. Will ignore it.`);
+            continue;
+        }
+        const flowKey = action.interfaceAlias || verb;
+        const flowParam = action.paramSchema ? dumpActionParam(plan.param || {}, action.paramSchema) : {};
+        const flowItem = {
+            [flowKey]: '',
+            ...flowParam
+        };
+        flow.push(flowItem);
+    }
+    if (sleep) flow.push({
+        sleep
+    });
+    return flow;
+}
+const PointSchema = external_zod_namespaceObject.z.object({
+    left: external_zod_namespaceObject.z.number(),
+    top: external_zod_namespaceObject.z.number()
+});
+const SizeSchema = external_zod_namespaceObject.z.object({
+    width: external_zod_namespaceObject.z.number(),
+    height: external_zod_namespaceObject.z.number(),
+    dpr: external_zod_namespaceObject.z.number().optional()
+});
+const RectSchema = PointSchema.and(SizeSchema).and(external_zod_namespaceObject.z.object({
+    zoom: external_zod_namespaceObject.z.number().optional()
+}));
+const TMultimodalPromptSchema = external_zod_namespaceObject.z.object({
+    images: external_zod_namespaceObject.z.array(external_zod_namespaceObject.z.object({
+        name: external_zod_namespaceObject.z.string(),
+        url: external_zod_namespaceObject.z.string()
+    })).optional(),
+    convertHttpImage2Base64: external_zod_namespaceObject.z.boolean().optional()
+});
+const TUserPromptSchema = external_zod_namespaceObject.z.union([
+    external_zod_namespaceObject.z.string(),
+    external_zod_namespaceObject.z.object({
+        prompt: external_zod_namespaceObject.z.string()
+    }).and(TMultimodalPromptSchema.partial())
+]);
+const locateFieldFlagName = 'rpascene_location_field_flag';
+const RpasceneLocationInput = external_zod_namespaceObject.z.object({
+    prompt: TUserPromptSchema,
+    deepThink: external_zod_namespaceObject.z.boolean().optional(),
+    cacheable: external_zod_namespaceObject.z.boolean().optional(),
+    xpath: external_zod_namespaceObject.z.union([
+        external_zod_namespaceObject.z.string(),
+        external_zod_namespaceObject.z.boolean()
+    ]).optional()
+}).passthrough();
+external_zod_namespaceObject.z.object({
+    [locateFieldFlagName]: external_zod_namespaceObject.z.literal(true),
+    prompt: TUserPromptSchema,
+    deepThink: external_zod_namespaceObject.z.boolean().optional(),
+    cacheable: external_zod_namespaceObject.z.boolean().optional(),
+    xpath: external_zod_namespaceObject.z.boolean().optional(),
+    center: external_zod_namespaceObject.z.tuple([
+        external_zod_namespaceObject.z.number(),
+        external_zod_namespaceObject.z.number()
+    ]),
+    rect: RectSchema
+}).passthrough();
+const getRpasceneLocationSchema = ()=>RpasceneLocationInput;
+const ifRpasceneLocatorField = (field)=>{
+    var _actualField__def, _actualField__def1;
+    let actualField = field;
+    if ((null == (_actualField__def = actualField._def) ? void 0 : _actualField__def.typeName) === 'ZodOptional') actualField = actualField._def.innerType;
+    if ((null == (_actualField__def1 = actualField._def) ? void 0 : _actualField__def1.typeName) === 'ZodObject') {
+        const shape = actualField._def.shape();
+        if (locateFieldFlagName in shape) return true;
+        if ('prompt' in shape && shape.prompt) return true;
+    }
+    return false;
+};
+const dumpRpasceneLocatorField = (field)=>{
+    (0, utils_namespaceObject.assert)(ifRpasceneLocatorField(field), 'field is not a rpascene locator field');
+    if ('string' == typeof field) return field;
+    if (field && 'object' == typeof field && field.prompt) {
+        if ('string' == typeof field.prompt) return field.prompt;
+        if ('object' == typeof field.prompt && field.prompt.prompt) return field.prompt.prompt;
+    }
+    return String(field);
+};
+const findAllRpasceneLocatorField = (zodType, requiredOnly)=>{
+    var _zodObject__def;
+    if (!zodType) return [];
+    const zodObject = zodType;
+    if ((null == (_zodObject__def = zodObject._def) ? void 0 : _zodObject__def.typeName) === 'ZodObject' && zodObject.shape) {
+        const keys = Object.keys(zodObject.shape);
+        return keys.filter((key)=>{
+            const field = zodObject.shape[key];
+            if (!ifRpasceneLocatorField(field)) return false;
+            if (requiredOnly) {
+                var _field__def;
+                return (null == (_field__def = field._def) ? void 0 : _field__def.typeName) !== 'ZodOptional';
+            }
+            return true;
+        });
+    }
+    return [];
+};
+const dumpActionParam = (jsonObject, zodSchema)=>{
+    const locatorFields = findAllRpasceneLocatorField(zodSchema);
+    const result = {
+        ...jsonObject
+    };
+    for (const fieldName of locatorFields){
+        const fieldValue = result[fieldName];
+        if (fieldValue) {
+            if ('string' == typeof fieldValue) result[fieldName] = fieldValue;
+            else if ('object' == typeof fieldValue) {
+                if (fieldValue.prompt) {
+                    if ('string' == typeof fieldValue.prompt) result[fieldName] = fieldValue.prompt;
+                    else if ('object' == typeof fieldValue.prompt && fieldValue.prompt.prompt) result[fieldName] = fieldValue.prompt.prompt;
+                }
+            }
+        }
+    }
+    return result;
+};
+const loadActionParam = (jsonObject, zodSchema)=>{
+    const locatorFields = findAllRpasceneLocatorField(zodSchema);
+    const result = {
+        ...jsonObject
+    };
+    for (const fieldName of locatorFields){
+        const fieldValue = result[fieldName];
+        if (fieldValue && 'string' == typeof fieldValue) result[fieldName] = {
+            [locateFieldFlagName]: true,
+            prompt: fieldValue
+        };
+    }
+    return result;
+};
+const parseActionParam = (rawParam, zodSchema)=>{
+    const param = rawParam ?? {};
+    const locateFields = findAllRpasceneLocatorField(zodSchema);
+    if (0 === locateFields.length) return zodSchema.parse(param);
+    const locateFieldValues = {};
+    for (const fieldName of locateFields)if (fieldName in param) locateFieldValues[fieldName] = param[fieldName];
+    const paramsForValidation = {};
+    for(const key in param)if (locateFields.includes(key)) paramsForValidation[key] = {
+        prompt: '_dummy_'
+    };
+    else paramsForValidation[key] = param[key];
+    const validated = zodSchema.parse(paramsForValidation);
+    for(const fieldName in locateFieldValues)validated[fieldName] = locateFieldValues[fieldName];
+    return validated;
+};
+exports.AIActionType = __webpack_exports__.AIActionType;
+exports.PointSchema = __webpack_exports__.PointSchema;
+exports.RectSchema = __webpack_exports__.RectSchema;
+exports.SizeSchema = __webpack_exports__.SizeSchema;
+exports.TMultimodalPromptSchema = __webpack_exports__.TMultimodalPromptSchema;
+exports.TUserPromptSchema = __webpack_exports__.TUserPromptSchema;
+exports.adaptBbox = __webpack_exports__.adaptBbox;
+exports.adaptBboxToRect = __webpack_exports__.adaptBboxToRect;
+exports.adaptDoubaoBbox = __webpack_exports__.adaptDoubaoBbox;
+exports.adaptGeminiBbox = __webpack_exports__.adaptGeminiBbox;
+exports.adaptQwenBbox = __webpack_exports__.adaptQwenBbox;
+exports.buildYamlFlowFromPlans = __webpack_exports__.buildYamlFlowFromPlans;
+exports.dumpActionParam = __webpack_exports__.dumpActionParam;
+exports.dumpRpasceneLocatorField = __webpack_exports__.dumpRpasceneLocatorField;
+exports.expandSearchArea = __webpack_exports__.expandSearchArea;
+exports.fillBboxParam = __webpack_exports__.fillBboxParam;
+exports.findAllRpasceneLocatorField = __webpack_exports__.findAllRpasceneLocatorField;
+exports.getRpasceneLocationSchema = __webpack_exports__.getRpasceneLocationSchema;
+exports.ifRpasceneLocatorField = __webpack_exports__.ifRpasceneLocatorField;
+exports.loadActionParam = __webpack_exports__.loadActionParam;
+exports.markupImageForLLM = __webpack_exports__.markupImageForLLM;
+exports.mergeRects = __webpack_exports__.mergeRects;
+exports.normalized01000 = __webpack_exports__.normalized01000;
+exports.parseActionParam = __webpack_exports__.parseActionParam;
+exports.warnGPT4oSizeLimit = __webpack_exports__.warnGPT4oSizeLimit;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "AIActionType",
+    "PointSchema",
+    "RectSchema",
+    "SizeSchema",
+    "TMultimodalPromptSchema",
+    "TUserPromptSchema",
+    "adaptBbox",
+    "adaptBboxToRect",
+    "adaptDoubaoBbox",
+    "adaptGeminiBbox",
+    "adaptQwenBbox",
+    "buildYamlFlowFromPlans",
+    "dumpActionParam",
+    "dumpRpasceneLocatorField",
+    "expandSearchArea",
+    "fillBboxParam",
+    "findAllRpasceneLocatorField",
+    "getRpasceneLocationSchema",
+    "ifRpasceneLocatorField",
+    "loadActionParam",
+    "markupImageForLLM",
+    "mergeRects",
+    "normalized01000",
+    "parseActionParam",
+    "warnGPT4oSizeLimit"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=common.js.map