@midscene/core 0.26.5-beta-20250814095614.0 → 0.26.5

package/dist/es/ai-model/action-executor.mjs

@@ -0,0 +1,139 @@
+import { getVersion } from "../utils.mjs";
+import { MIDSCENE_MODEL_NAME, getAIConfig, uiTarsModelVersion, vlLocateMode } from "@midscene/shared/env";
+import { assert } from "@midscene/shared/utils";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class Executor {
+    markTaskAsPending(task) {
+        return {
+            status: 'pending',
+            ...task
+        };
+    }
+    async append(task) {
+        var _this_latestErrorTask, _this_latestErrorTask1;
+        assert('error' !== this.status, `executor is in error state, cannot append task\nerror=${null == (_this_latestErrorTask = this.latestErrorTask()) ? void 0 : _this_latestErrorTask.error}\n${null == (_this_latestErrorTask1 = this.latestErrorTask()) ? void 0 : _this_latestErrorTask1.errorStack}`);
+        if (Array.isArray(task)) this.tasks.push(...task.map((item)=>this.markTaskAsPending(item)));
+        else this.tasks.push(this.markTaskAsPending(task));
+        if ('running' !== this.status) this.status = 'pending';
+    }
+    async flush() {
+        if ('init' === this.status && this.tasks.length > 0) console.warn('illegal state for executor, status is init but tasks are not empty');
+        assert('running' !== this.status, 'executor is already running');
+        assert('completed' !== this.status, 'executor is already completed');
+        assert('error' !== this.status, 'executor is in error state');
+        const nextPendingIndex = this.tasks.findIndex((task)=>'pending' === task.status);
+        if (nextPendingIndex < 0) return;
+        this.status = 'running';
+        let taskIndex = nextPendingIndex;
+        let successfullyCompleted = true;
+        let previousFindOutput;
+        while(taskIndex < this.tasks.length){
+            const task = this.tasks[taskIndex];
+            assert('pending' === task.status, `task status should be pending, but got: ${task.status}`);
+            task.timing = {
+                start: Date.now()
+            };
+            try {
+                task.status = 'running';
+                try {
+                    if (this.onTaskStart) await this.onTaskStart(task);
+                } catch (e) {
+                    console.error('error in onTaskStart', e);
+                }
+                assert([
+                    'Insight',
+                    'Action',
+                    'Planning'
+                ].indexOf(task.type) >= 0, `unsupported task type: ${task.type}`);
+                const { executor, param } = task;
+                assert(executor, `executor is required for task type: ${task.type}`);
+                let returnValue;
+                const executorContext = {
+                    task,
+                    element: null == previousFindOutput ? void 0 : previousFindOutput.element
+                };
+                if ('Insight' === task.type) {
+                    assert('Locate' === task.subType || 'Query' === task.subType || 'Assert' === task.subType || 'Boolean' === task.subType || 'Number' === task.subType || 'String' === task.subType, `unsupported insight subType: ${task.subType}`);
+                    returnValue = await task.executor(param, executorContext);
+                    if ('Locate' === task.subType) previousFindOutput = null == returnValue ? void 0 : returnValue.output;
+                } else if ('Action' === task.type || 'Planning' === task.type) returnValue = await task.executor(param, executorContext);
+                else {
+                    console.warn(`unsupported task type: ${task.type}, will try to execute it directly`);
+                    returnValue = await task.executor(param, executorContext);
+                }
+                Object.assign(task, returnValue);
+                task.status = 'finished';
+                task.timing.end = Date.now();
+                task.timing.cost = task.timing.end - task.timing.start;
+                taskIndex++;
+            } catch (e) {
+                successfullyCompleted = false;
+                task.error = e;
+                task.errorMessage = (null == e ? void 0 : e.message) || ('string' == typeof e ? e : 'error-without-message');
+                task.errorStack = e.stack;
+                task.status = 'failed';
+                task.timing.end = Date.now();
+                task.timing.cost = task.timing.end - task.timing.start;
+                break;
+            }
+        }
+        for(let i = taskIndex + 1; i < this.tasks.length; i++)this.tasks[i].status = 'cancelled';
+        if (successfullyCompleted) this.status = 'completed';
+        else this.status = 'error';
+        if (this.tasks.length) {
+            const outputIndex = Math.min(taskIndex, this.tasks.length - 1);
+            const { thought, output } = this.tasks[outputIndex];
+            return {
+                thought,
+                output
+            };
+        }
+    }
+    isInErrorState() {
+        return 'error' === this.status;
+    }
+    latestErrorTask() {
+        if ('error' !== this.status) return null;
+        const errorTaskIndex = this.tasks.findIndex((task)=>'failed' === task.status);
+        if (errorTaskIndex >= 0) return this.tasks[errorTaskIndex];
+        return null;
+    }
+    dump() {
+        let modelDescription = '';
+        if (vlLocateMode()) {
+            const uiTarsModelVer = uiTarsModelVersion();
+            modelDescription = uiTarsModelVer ? `UI-TARS=${uiTarsModelVer}` : `${vlLocateMode()} mode`;
+        }
+        const dumpData = {
+            sdkVersion: getVersion(),
+            model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',
+            model_description: modelDescription,
+            logTime: Date.now(),
+            name: this.name,
+            tasks: this.tasks
+        };
+        return dumpData;
+    }
+    constructor(name, options){
+        _define_property(this, "name", void 0);
+        _define_property(this, "tasks", void 0);
+        _define_property(this, "status", void 0);
+        _define_property(this, "onTaskStart", void 0);
+        this.status = (null == options ? void 0 : options.tasks) && options.tasks.length > 0 ? 'pending' : 'init';
+        this.name = name;
+        this.tasks = ((null == options ? void 0 : options.tasks) || []).map((item)=>this.markTaskAsPending(item));
+        this.onTaskStart = null == options ? void 0 : options.onTaskStart;
+    }
+}
+export { Executor };
+
+//# sourceMappingURL=action-executor.mjs.map

package/dist/es/ai-model/action-executor.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/action-executor.mjs","sources":["webpack://@midscene/core/./src/ai-model/action-executor.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport {\n  MIDSCENE_MODEL_NAME,\n  getAIConfig,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    let modelDescription = '';\n\n    if (vlLocateMode()) {\n      const uiTarsModelVer = uiTarsModelVersion();\n      if (uiTarsModelVer) {\n        modelDescription = `UI-TARS=${uiTarsModelVer}`;\n      } else {\n        modelDescription = `${vlLocateMode()} mode`;\n      }\n    }\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      model_description: modelDescription,\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n"],"names":["Executor","task","_this_latestErrorTask","_this_latestErrorTask1","assert","Array","item","console","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","Date","e","executor","param","returnValue","executorContext","Object","i","outputIndex","Math","thought","output","errorTaskIndex","modelDescription","vlLocateMode","uiTarsModelVer","uiTarsModelVersion","dumpData","getVersion","getAIConfig","MIDSCENE_MODEL_NAME","name","options"],"mappings":";;;;;;;;;;;;;AAkBO,MAAMA;IAyBH,kBAAkBC,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQ;YACR,GAAGA,IAAI;QACT;IACF;IAEA,MAAM,OAAOA,IAA+C,EAAiB;YAGhBC,uBAAkCC;QAF7FC,OACE,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACX,CAAC,sDAAsD,EAAE,QAAAF,CAAAA,wBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,sBAAwB,KAAK,CAAC,EAAE,EAAE,QAAAC,CAAAA,yBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,uBAAwB,UAAU,EAAE;QAEjI,IAAIE,MAAM,OAAO,CAACJ,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACK,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACL;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;IAElB;IAEA,MAAM,QAAgE;QACpE,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDM,QAAQ,IAAI,CACV;QAIJH,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QACpCA,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM,EAAc;QAEhC,MAAMI,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACP,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIO,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMR,OAAO,IAAI,CAAC,KAAK,CAACQ,UAAU;YAClCL,OACEH,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAOW,KAAK,GAAG;YACjB;YACA,IAAI;gBACFX,KAAK,MAAM,GAAG;gBACd,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOY,GAAG;oBACVN,QAAQ,KAAK,CAAC,wBAAwBM;gBACxC;gBACAT,OACE;oBAAC;oBAAW;oBAAU;iBAAW,CAAC,OAAO,CAACH,KAAK,IAAI,KAAK,GACxD,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEa,QAAQ,EAAEC,KAAK,EAAE,GAAGd;gBAC5BG,OAAOU,UAAU,CAAC,oCAAoC,EAAEb,KAAK,IAAI,EAAE;gBAEnE,IAAIe;gBACJ,MAAMC,kBAAmC;oBACvChB;oBACA,SAASU,QAAAA,qBAAAA,KAAAA,IAAAA,mBAAoB,OAAO;gBACtC;gBAEA,IAAIV,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BG,OACEH,AAAiB,aAAjBA,KAAK,OAAO,IACVA,AAAiB,YAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDe,cAAc,MAAMf,KAAK,QAAQ,CAACc,OAAOE;oBACzC,IAAIhB,AAAiB,aAAjBA,KAAK,OAAO,EACdU,qBACEK,QAAAA,cAAAA,KAAAA,IAAAA,YACC,MAAM;gBAEb,OAAO,IAAIf,AAAc,aAAdA,KAAK,IAAI,IAAiBA,AAAc,eAAdA,KAAK,IAAI,EAC5Ce,cAAc,MAAMf,KAAK,QAAQ,CAACc,OAAOE;qBACpC;oBACLV,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEN,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEe,cAAc,MAAMf,KAAK,QAAQ,CAACc,OAAOE;gBAC3C;gBAEAC,OAAO,MAAM,CAACjB,MAAMe;gBACpBf,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGW,KAAK,GAAG;gBAC1BX,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtDQ;YACF,EAAE,OAAOI,GAAQ;gBACfH,wBAAwB;gBACxBT,KAAK,KAAK,GAAGY;gBACbZ,KAAK,YAAY,GACfY,AAAAA,CAAAA,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO,AAAD,KAAM,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEZ,KAAK,UAAU,GAAGY,EAAE,KAAK;gBAEzBZ,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGW,KAAK,GAAG;gBAC1BX,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD;YACF;QACF;QAGA,IAAK,IAAIkB,IAAIV,YAAY,GAAGU,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAGzB,IAAIT,uBACF,IAAI,CAAC,MAAM,GAAG;aAEd,IAAI,CAAC,MAAM,GAAG;QAGhB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMU,cAAcC,KAAK,GAAG,CAACZ,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEa,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAET,MAAMC,iBAAiB,IAAI,CAAC,KAAK,CAAC,SAAS,CACzC,CAACvB,OAASA,AAAgB,aAAhBA,KAAK,MAAM;QAEvB,IAAIuB,kBAAkB,GACpB,OAAO,IAAI,CAAC,KAAK,CAACA,eAAe;QAEnC,OAAO;IACT;IAEA,OAAsB;QACpB,IAAIC,mBAAmB;QAEvB,IAAIC,gBAAgB;YAClB,MAAMC,iBAAiBC;YAErBH,mBADEE,iBACiB,CAAC,QAAQ,EAAEA,gBAAgB,GAE3B,GAAGD,eAAe,KAAK,CAAC;QAE/C;QACA,MAAMG,WAA0B;YAC9B,YAAYC;YACZ,YAAYC,YAAYC,wBAAwB;YAChD,mBAAmBP;YACnB,SAASb,KAAK,GAAG;YACjB,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;QACA,OAAOiB;IACT;IArMA,YACEI,IAAY,EACZC,OAEC,CACD;QAdF;QAEA;QAGA;QAEA;QAQE,IAAI,CAAC,MAAM,GACTA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAKA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAGD;QACZ,IAAI,CAAC,KAAK,GAAIC,AAAAA,CAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAK,EAAC,EAAG,GAAG,CAAC,CAAC5B,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAG4B,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,WAAW;IACzC;AAyLF"}

package/dist/es/ai-model/common.mjs

@@ -0,0 +1,219 @@
+import { assert } from "@midscene/shared/utils";
+import { callToGetJSONObject, getModelName } from "./service-caller/index.mjs";
+import { NodeType } from "@midscene/shared/constants";
+import { vlLocateMode } from "@midscene/shared/env";
+import { treeToList } from "@midscene/shared/extractor";
+import { compositeElementInfoImg } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+var common_AIActionType = /*#__PURE__*/ function(AIActionType) {
+    AIActionType[AIActionType["ASSERT"] = 0] = "ASSERT";
+    AIActionType[AIActionType["INSPECT_ELEMENT"] = 1] = "INSPECT_ELEMENT";
+    AIActionType[AIActionType["EXTRACT_DATA"] = 2] = "EXTRACT_DATA";
+    AIActionType[AIActionType["PLAN"] = 3] = "PLAN";
+    AIActionType[AIActionType["DESCRIBE_ELEMENT"] = 4] = "DESCRIBE_ELEMENT";
+    return AIActionType;
+}({});
+async function callAiFn(msgs, AIActionTypeValue) {
+    const jsonObject = await callToGetJSONObject(msgs, AIActionTypeValue);
+    return {
+        content: jsonObject.content,
+        usage: jsonObject.usage
+    };
+}
+const defaultBboxSize = 20;
+const debugInspectUtils = getDebug('ai:common');
+function fillBboxParam(locate, width, height) {
+    if (locate.bbox_2d && !(null == locate ? void 0 : locate.bbox)) {
+        locate.bbox = locate.bbox_2d;
+        delete locate.bbox_2d;
+    }
+    if (null == locate ? void 0 : locate.bbox) locate.bbox = adaptBbox(locate.bbox, width, height);
+    return locate;
+}
+function adaptQwenBbox(bbox) {
+    if (bbox.length < 2) {
+        const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;
+        throw new Error(msg);
+    }
+    const result = [
+        Math.round(bbox[0]),
+        Math.round(bbox[1]),
+        'number' == typeof bbox[2] ? Math.round(bbox[2]) : Math.round(bbox[0] + defaultBboxSize),
+        'number' == typeof bbox[3] ? Math.round(bbox[3]) : Math.round(bbox[1] + defaultBboxSize)
+    ];
+    return result;
+}
+function adaptDoubaoBbox(bbox, width, height) {
+    assert(width > 0 && height > 0, 'width and height must be greater than 0 in doubao mode');
+    if ('string' == typeof bbox) {
+        assert(/^(\d+)\s(\d+)\s(\d+)\s(\d+)$/.test(bbox.trim()), `invalid bbox data string for doubao-vision mode: ${bbox}`);
+        const splitted = bbox.split(' ');
+        if (4 === splitted.length) return [
+            Math.round(Number(splitted[0]) * width / 1000),
+            Math.round(Number(splitted[1]) * height / 1000),
+            Math.round(Number(splitted[2]) * width / 1000),
+            Math.round(Number(splitted[3]) * height / 1000)
+        ];
+        throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);
+    }
+    if (Array.isArray(bbox) && Array.isArray(bbox[0])) bbox = bbox[0];
+    let bboxList = [];
+    if (Array.isArray(bbox) && 'string' == typeof bbox[0]) bbox.forEach((item)=>{
+        if ('string' == typeof item && item.includes(',')) {
+            const [x, y] = item.split(',');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else if ('string' == typeof item && item.includes(' ')) {
+            const [x, y] = item.split(' ');
+            bboxList.push(Number(x.trim()), Number(y.trim()));
+        } else bboxList.push(Number(item));
+    });
+    else bboxList = bbox;
+    if (4 === bboxList.length || 5 === bboxList.length) return [
+        Math.round(bboxList[0] * width / 1000),
+        Math.round(bboxList[1] * height / 1000),
+        Math.round(bboxList[2] * width / 1000),
+        Math.round(bboxList[3] * height / 1000)
+    ];
+    if (6 === bboxList.length || 2 === bboxList.length || 3 === bboxList.length || 7 === bboxList.length) return [
+        Math.max(0, Math.round(bboxList[0] * width / 1000) - defaultBboxSize / 2),
+        Math.max(0, Math.round(bboxList[1] * height / 1000) - defaultBboxSize / 2),
+        Math.min(width, Math.round(bboxList[0] * width / 1000) + defaultBboxSize / 2),
+        Math.min(height, Math.round(bboxList[1] * height / 1000) + defaultBboxSize / 2)
+    ];
+    if (8 === bbox.length) return [
+        Math.round(bboxList[0] * width / 1000),
+        Math.round(bboxList[1] * height / 1000),
+        Math.round(bboxList[4] * width / 1000),
+        Math.round(bboxList[5] * height / 1000)
+    ];
+    const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;
+    throw new Error(msg);
+}
+function adaptBbox(bbox, width, height) {
+    if ('doubao-vision' === vlLocateMode() || 'vlm-ui-tars' === vlLocateMode()) return adaptDoubaoBbox(bbox, width, height);
+    if ('gemini' === vlLocateMode()) return adaptGeminiBbox(bbox, width, height);
+    return adaptQwenBbox(bbox);
+}
+function adaptGeminiBbox(bbox, width, height) {
+    const left = Math.round(bbox[1] * width / 1000);
+    const top = Math.round(bbox[0] * height / 1000);
+    const right = Math.round(bbox[3] * width / 1000);
+    const bottom = Math.round(bbox[2] * height / 1000);
+    return [
+        left,
+        top,
+        right,
+        bottom
+    ];
+}
+function adaptBboxToRect(bbox, width, height, offsetX = 0, offsetY = 0) {
+    debugInspectUtils('adaptBboxToRect', bbox, width, height, offsetX, offsetY);
+    const [left, top, right, bottom] = adaptBbox(bbox, width, height);
+    const rect = {
+        left: left + offsetX,
+        top: top + offsetY,
+        width: right - left,
+        height: bottom - top
+    };
+    debugInspectUtils('adaptBboxToRect, result=', rect);
+    return rect;
+}
+let warned = false;
+function warnGPT4oSizeLimit(size) {
+    var _getModelName;
+    if (warned) return;
+    if (null == (_getModelName = getModelName()) ? void 0 : _getModelName.toLowerCase().includes('gpt-4o')) {
+        const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your page to a smaller resolution. Otherwise, the result may be inaccurate.`;
+        if (Math.max(size.width, size.height) > 2000 || Math.min(size.width, size.height) > 768) {
+            console.warn(warningMsg);
+            warned = true;
+        }
+    } else if (size.width > 1800 || size.height > 1800) {
+        console.warn(`The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`);
+        warned = true;
+    }
+}
+function mergeRects(rects) {
+    const minLeft = Math.min(...rects.map((r)=>r.left));
+    const minTop = Math.min(...rects.map((r)=>r.top));
+    const maxRight = Math.max(...rects.map((r)=>r.left + r.width));
+    const maxBottom = Math.max(...rects.map((r)=>r.top + r.height));
+    return {
+        left: minLeft,
+        top: minTop,
+        width: maxRight - minLeft,
+        height: maxBottom - minTop
+    };
+}
+function expandSearchArea(rect, screenSize) {
+    const minEdgeSize = 'doubao-vision' === vlLocateMode() ? 500 : 300;
+    const defaultPadding = 160;
+    const paddingSizeHorizontal = rect.width < minEdgeSize ? Math.ceil((minEdgeSize - rect.width) / 2) : defaultPadding;
+    const paddingSizeVertical = rect.height < minEdgeSize ? Math.ceil((minEdgeSize - rect.height) / 2) : defaultPadding;
+    rect.left = Math.max(0, rect.left - paddingSizeHorizontal);
+    rect.width = Math.min(rect.width + 2 * paddingSizeHorizontal, screenSize.width - rect.left);
+    rect.top = Math.max(0, rect.top - paddingSizeVertical);
+    rect.height = Math.min(rect.height + 2 * paddingSizeVertical, screenSize.height - rect.top);
+    return rect;
+}
+async function markupImageForLLM(screenshotBase64, tree, size) {
+    const elementsInfo = treeToList(tree);
+    const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo)=>{
+        if (elementInfo.attributes.nodeType === NodeType.TEXT) return false;
+        return true;
+    });
+    const imagePayload = await compositeElementInfoImg({
+        inputImgBase64: screenshotBase64,
+        elementsPositionInfo: elementsPositionInfoWithoutText,
+        size
+    });
+    return imagePayload;
+}
+function buildYamlFlowFromPlans(plans, sleep) {
+    const flow = [];
+    for (const plan of plans){
+        var _plan_locate;
+        const type = plan.type;
+        const locate = null == (_plan_locate = plan.locate) ? void 0 : _plan_locate.prompt;
+        if ('Tap' === type) flow.push({
+            aiTap: locate
+        });
+        else if ('Hover' === type) flow.push({
+            aiHover: locate
+        });
+        else if ('Input' === type) {
+            const param = plan.param;
+            flow.push({
+                aiInput: param.value,
+                locate
+            });
+        } else if ('KeyboardPress' === type) {
+            const param = plan.param;
+            flow.push({
+                aiKeyboardPress: param.value,
+                locate
+            });
+        } else if ('Scroll' === type) {
+            const param = plan.param;
+            flow.push({
+                aiScroll: null,
+                locate,
+                direction: param.direction,
+                scrollType: param.scrollType,
+                distance: param.distance
+            });
+        } else if ('Sleep' === type) {
+            const param = plan.param;
+            flow.push({
+                sleep: param.timeMs
+            });
+        } else 'AndroidBackButton' === type || 'AndroidHomeButton' === type || 'AndroidRecentAppsButton' === type || 'AndroidLongPress' === type || 'AndroidPull' === type || 'Error' === type || 'Assert' === type || 'AssertWithoutThrow' === type || 'Finished' === type || console.warn(`Cannot convert action ${type} to yaml flow. This should be a bug of Midscene.`);
+    }
+    if (sleep) flow.push({
+        sleep: sleep
+    });
+    return flow;
+}
+export { common_AIActionType as AIActionType, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptQwenBbox, buildYamlFlowFromPlans, callAiFn, expandSearchArea, fillBboxParam, markupImageForLLM, mergeRects, warnGPT4oSizeLimit };
+
+//# sourceMappingURL=common.mjs.map

package/dist/es/ai-model/common.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  PlanningActionParamInputOrKeyPress,\n  PlanningActionParamSleep,\n  Rect,\n  ScrollParam,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport {\n  call,\n  callToGetJSONObject,\n  getModelName,\n} from './service-caller/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport { vlLocateMode } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n}\n\nexport async function callAiFn<T>(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const jsonObject = await callToGetJSONObject<T>(msgs, AIActionTypeValue);\n\n  return {\n    content: jsonObject.content,\n    usage: jsonObject.usage,\n  };\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(locate.bbox, width, height);\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  if (vlLocateMode() === 'doubao-vision' || vlLocateMode() === 'vlm-ui-tars') {\n    return adaptDoubaoBbox(bbox, width, height);\n  }\n\n  if (vlLocateMode() === 'gemini') {\n    return adaptGeminiBbox(bbox, width, height);\n  }\n\n  return adaptQwenBbox(bbox);\n}\n\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n): Rect {\n  debugInspectUtils('adaptBboxToRect', bbox, width, height, offsetX, offsetY);\n  const [left, top, right, bottom] = adaptBbox(bbox, width, height);\n  const rect = {\n    left: left + offsetX,\n    top: top + offsetY,\n    width: right - left,\n    height: bottom - top,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(size: Size) {\n  if (warned) return;\n  if (getModelName()?.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your page to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(rect: Rect, screenSize: Size) {\n  const minEdgeSize = vlLocateMode() === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n  rect.left = Math.max(0, rect.left - paddingSizeHorizontal);\n  rect.width = Math.min(\n    rect.width + paddingSizeHorizontal * 2,\n    screenSize.width - rect.left,\n  );\n  rect.top = Math.max(0, rect.top - paddingSizeVertical);\n  rect.height = Math.min(\n    rect.height + paddingSizeVertical * 2,\n    screenSize.height - rect.top,\n  );\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const type = plan.type;\n    const locate = plan.locate?.prompt!; // TODO: check if locate is null\n\n    if (type === 'Tap') {\n      flow.push({\n        aiTap: locate!,\n      });\n    } else if (type === 'Hover') {\n      flow.push({\n        aiHover: locate!,\n      });\n    } else if (type === 'Input') {\n      const param = plan.param as PlanningActionParamInputOrKeyPress;\n      flow.push({\n        aiInput: param.value,\n        locate,\n      });\n    } else if (type === 'KeyboardPress') {\n      const param = plan.param as PlanningActionParamInputOrKeyPress;\n      flow.push({\n        aiKeyboardPress: param.value,\n        locate,\n      });\n    } else if (type === 'Scroll') {\n      const param = plan.param as ScrollParam;\n      flow.push({\n        aiScroll: null,\n        locate,\n        direction: param.direction,\n        scrollType: param.scrollType,\n        distance: param.distance,\n      });\n    } else if (type === 'Sleep') {\n      const param = plan.param as PlanningActionParamSleep;\n      flow.push({\n        sleep: param.timeMs,\n      });\n    } else if (\n      type === 'AndroidBackButton' ||\n      type === 'AndroidHomeButton' ||\n      type === 'AndroidRecentAppsButton' ||\n      type === 'AndroidLongPress' ||\n      type === 'AndroidPull'\n    ) {\n      // not implemented in yaml yet\n    } else if (\n      type === 'Error' ||\n      type === 'Assert' ||\n      type === 'AssertWithoutThrow' ||\n      type === 'Finished'\n    ) {\n      // do nothing\n    } else {\n      console.warn(\n        `Cannot convert action ${type} to yaml flow. This should be a bug of Midscene.`,\n      );\n    }\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep: sleep,\n    });\n  }\n\n  return flow;\n}\n"],"names":["AIActionType","callAiFn","msgs","AIActionTypeValue","jsonObject","callToGetJSONObject","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","vlLocateMode","adaptGeminiBbox","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rect","warned","warnGPT4oSizeLimit","size","_getModelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","sleep","flow","plan","_plan_locate","type","param"],"mappings":";;;;;;;AAoCO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;WAAZA;;AAQL,eAAeC,SACpBC,IAAY,EACZC,iBAA+B;IAE/B,MAAMC,aAAa,MAAMC,oBAAuBH,MAAMC;IAEtD,OAAO;QACL,SAASC,WAAW,OAAO;QAC3B,OAAOA,WAAW,KAAK;IACzB;AACF;AAEA,MAAME,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc;IAGd,IAAKF,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGG,UAAUH,OAAO,IAAI,EAAEC,OAAOC;IAG9C,OAAOF;AACT;AAEO,SAASI,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;QACN,YAAnB,OAAOS,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;KAC1B;IACD,OAAOa;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCJ,KAAa,EACbC,MAAc;IAEdU,OACEX,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOG,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;YAC5CQ,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;SAC7C;QAEH,MAAM,IAAIM,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAIH,IACEc,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;QAEhEc,KAAK,GAAG,CACNT,OACAS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACNR,QACAQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIS,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAGH,MAAMI,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdJ,KAAa,EACbC,MAAc;IAEd,IAAIkB,AAAmB,oBAAnBA,kBAAsCA,AAAmB,kBAAnBA,gBACxC,OAAOT,gBAAgBN,MAAMJ,OAAOC;IAGtC,IAAIkB,AAAmB,aAAnBA,gBACF,OAAOC,gBAAgBhB,MAAMJ,OAAOC;IAGtC,OAAOE,cAAcC;AACvB;AAEO,SAASgB,gBACdhB,IAAc,EACdJ,KAAa,EACbC,MAAc;IAEd,MAAMoB,OAAOZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC5C,MAAMsB,MAAMb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC5C,MAAMsB,QAAQd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC7C,MAAMwB,SAASf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC/C,OAAO;QAACoB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdrB,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdyB,UAAU,CAAC,EACXC,UAAU,CAAC;IAEX/B,kBAAkB,mBAAmBQ,MAAMJ,OAAOC,QAAQyB,SAASC;IACnE,MAAM,CAACN,MAAMC,KAAKC,OAAOC,OAAO,GAAGtB,UAAUE,MAAMJ,OAAOC;IAC1D,MAAM2B,OAAO;QACX,MAAMP,OAAOK;QACb,KAAKJ,MAAMK;QACX,OAAOJ,QAAQF;QACf,QAAQG,SAASF;IACnB;IACA1B,kBAAkB,4BAA4BgC;IAC9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBAAmBC,IAAU;QAEvCC;IADJ,IAAIH,QAAQ;IACZ,IAAI,QAAAG,CAAAA,gBAAAA,cAAa,IAAbA,KAAAA,IAAAA,cAAgB,WAAW,GAAG,QAAQ,CAAC,WAAW;QACpD,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,wFAAwF,CAAC;QAEhN,IACEtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpCtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBAAiBd,IAAU,EAAEe,UAAgB;IAC3D,MAAMC,cAAczB,AAAmB,oBAAnBA,iBAAqC,MAAM;IAC/D,MAAM0B,iBAAiB;IAEvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IACNjB,KAAK,IAAI,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,IAAI,GAAGkB;IACpClB,KAAK,KAAK,GAAGnB,KAAK,GAAG,CACnBmB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA,uBACbH,WAAW,KAAK,GAAGf,KAAK,IAAI;IAE9BA,KAAK,GAAG,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,GAAG,GAAGmB;IAClCnB,KAAK,MAAM,GAAGnB,KAAK,GAAG,CACpBmB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA,qBACdJ,WAAW,MAAM,GAAGf,KAAK,GAAG;IAE9B,OAAOA;AACT;AAEO,eAAeoB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCnB,IAAU;IAEV,MAAMoB,eAAeC,WAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtBtB;IACF;IACA,OAAOyB;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQH,MAAO;YAETI;QADf,MAAMC,OAAOF,KAAK,IAAI;QACtB,MAAM/D,SAAS,QAAAgE,CAAAA,eAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,MAAM;QAElC,IAAIC,AAAS,UAATA,MACFH,KAAK,IAAI,CAAC;YACR,OAAO9D;QACT;aACK,IAAIiE,AAAS,YAATA,MACTH,KAAK,IAAI,CAAC;YACR,SAAS9D;QACX;aACK,IAAIiE,AAAS,YAATA,MAAkB;YAC3B,MAAMC,QAAQH,KAAK,KAAK;YACxBD,KAAK,IAAI,CAAC;gBACR,SAASI,MAAM,KAAK;gBACpBlE;YACF;QACF,OAAO,IAAIiE,AAAS,oBAATA,MAA0B;YACnC,MAAMC,QAAQH,KAAK,KAAK;YACxBD,KAAK,IAAI,CAAC;gBACR,iBAAiBI,MAAM,KAAK;gBAC5BlE;YACF;QACF,OAAO,IAAIiE,AAAS,aAATA,MAAmB;YAC5B,MAAMC,QAAQH,KAAK,KAAK;YACxBD,KAAK,IAAI,CAAC;gBACR,UAAU;gBACV9D;gBACA,WAAWkE,MAAM,SAAS;gBAC1B,YAAYA,MAAM,UAAU;gBAC5B,UAAUA,MAAM,QAAQ;YAC1B;QACF,OAAO,IAAID,AAAS,YAATA,MAAkB;YAC3B,MAAMC,QAAQH,KAAK,KAAK;YACxBD,KAAK,IAAI,CAAC;gBACR,OAAOI,MAAM,MAAM;YACrB;QACF,OACW,wBAATD,QACAA,AAAS,wBAATA,QACAA,AAAS,8BAATA,QACAA,AAAS,uBAATA,QACAA,AAAS,kBAATA,QAIAA,AAAS,YAATA,QACAA,AAAS,aAATA,QACAA,AAAS,yBAATA,QACAA,AAAS,eAATA,QAIA9B,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAE8B,KAAK,gDAAgD,CAAC;IAGrF;IAEA,IAAIJ,OACFC,KAAK,IAAI,CAAC;QACR,OAAOD;IACT;IAGF,OAAOC;AACT"}

package/dist/es/ai-model/index.mjs

@@ -0,0 +1,10 @@
+import { call, callAiFnWithStringResponse, callToGetJSONObject } from "./service-caller/index.mjs";
+import { systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
+import { describeUserPage, elementByPositionWithElementInfo } from "./prompt/util.mjs";
+import { generatePlaywrightTest, generatePlaywrightTestStream } from "./prompt/playwright-generator.mjs";
+import { generateYamlTest, generateYamlTestStream } from "./prompt/yaml-generator.mjs";
+import { AiAssert, AiExtractElementInfo, AiLocateElement, AiLocateSection } from "./inspect.mjs";
+import { plan } from "./llm-planning.mjs";
+import { AIActionType, adaptBboxToRect, callAiFn } from "./common.mjs";
+import { resizeImageForUiTars, vlmPlanning } from "./ui-tars-planning.mjs";
+export { AIActionType, AiAssert, AiExtractElementInfo, AiLocateElement, AiLocateSection, adaptBboxToRect, call as callAi, callAiFn, callAiFnWithStringResponse, callToGetJSONObject, describeUserPage, elementByPositionWithElementInfo, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, plan, resizeImageForUiTars, systemPromptToLocateElement, vlmPlanning };