@midscene/core 0.13.1 → 0.13.2-beta-20250401015137.0

package/dist/es/index.js

@@ -3,7 +3,7 @@ import {
   getLogDirByType,
   getVersion,
   setLogDir
-} from "./chunk-QKOUPDNQ.js";
+} from "./chunk-JYD7DE4D.js";
 import {
   AiAssert,
   AiExtractElementInfo,
@@ -13,14 +13,15 @@ import {
   describeUserPage,
   plan,
   transformElementPositionToId
-} from "./chunk-KUTJMPAQ.js";
+} from "./chunk-ZSMRRNLV.js";
 import {
+  MIDSCENE_FORCE_DEEP_THINK,
   MIDSCENE_MODEL_NAME,
   MIDSCENE_USE_VLM_UI_TARS,
   getAIConfig,
   getAIConfigInBoolean,
   vlLocateMode
-} from "./chunk-I2I7OU2E.js";
+} from "./chunk-UBGEKXK7.js";
 
 // src/ai-model/action-executor.ts
 import { assert } from "@midscene/shared/utils";
@@ -177,7 +178,8 @@ ${this.latestErrorTask()?.errorStack}`
 };
 
 // src/insight/index.ts
-import { assert as assert3, getDebug } from "@midscene/shared/utils";
+import { getDebug } from "@midscene/shared/logger";
+import { assert as assert3 } from "@midscene/shared/utils";
 
 // src/insight/utils.ts
 import { assert as assert2 } from "@midscene/shared/utils";
@@ -233,11 +235,22 @@ var Insight = class {
     const dumpSubscriber = this.onceDumpUpdatedFn;
     this.onceDumpUpdatedFn = void 0;
     let searchAreaPrompt = void 0;
-    if (typeof query === "object") {
-      searchAreaPrompt = query.searchArea;
-      if (!searchAreaPrompt && query.deepThink) {
-        searchAreaPrompt = query.prompt;
-      }
+    assert3(typeof query === "object", "query should be an object for locate");
+    searchAreaPrompt = query.searchArea;
+    const globalDeepThinkSwitch = getAIConfigInBoolean(
+      MIDSCENE_FORCE_DEEP_THINK
+    );
+    if (globalDeepThinkSwitch) {
+      debug("globalDeepThinkSwitch", globalDeepThinkSwitch);
+    }
+    if (!searchAreaPrompt && (query.deepThink || globalDeepThinkSwitch)) {
+      searchAreaPrompt = query.prompt;
+    }
+    if (searchAreaPrompt && !vlLocateMode()) {
+      console.warn(
+        'The "deepThink" feature is not supported with general purposed LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model'
+      );
+      searchAreaPrompt = void 0;
     }
     const context = await this.contextRetrieverFn("locate");
     let searchArea = void 0;
@@ -245,10 +258,6 @@ var Insight = class {
     let searchAreaUsage = void 0;
     let searchAreaResponse = void 0;
     if (searchAreaPrompt) {
-      assert3(
-        vlLocateMode(),
-        "locate with search area is not supported with general purposed LLM. Please use Midscene VL model. https://midscenejs.com/choose-a-model"
-      );
       searchAreaResponse = await AiLocateSection({
         context,
         sectionDescription: searchAreaPrompt
@@ -296,6 +305,7 @@ ${parseResult.errors.join("\n")}`;
       matchedRect: rect,
       data: null,
       taskInfo,
+      deepThink: !!searchArea,
       error: errorLog
     };
     const elements = [];

package/dist/es/index.js.map

@@ -1 +1 @@
-{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAWA,SAAS,cAAc;AAEhB,IAAM,WAAN,MAAe;AAAA,EAUpB,YACE,MACA,SAGA;AACA,SAAK,SACH,SAAS,SAAS,QAAQ,MAAM,SAAS,IAAI,YAAY;AAC3D,SAAK,OAAO;AACZ,SAAK,SAAS,SAAS,SAAS,CAAC,GAAG;AAAA,MAAI,CAAC,SACvC,KAAK,kBAAkB,IAAI;AAAA,IAC7B;AACA,SAAK,cAAc,SAAS;AAAA,EAC9B;AAAA,EAEQ,kBAAkB,MAAyC;AACjE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAgE;AAC3E;AAAA,MACE,KAAK,WAAW;AAAA,MAChB;AAAA,QAAyD,KAAK,gBAAgB,GAAG,KAAK;AAAA,EAAK,KAAK,gBAAgB,GAAG,UAAU;AAAA,IAC/H;AACA,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAK,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,SAAS,KAAK,kBAAkB,IAAI,CAAC,CAAC;AAAA,IACrE,OAAO;AACL,WAAK,MAAM,KAAK,KAAK,kBAAkB,IAAI,CAAC;AAAA,IAC9C;AACA,QAAI,KAAK,WAAW,WAAW;AAC7B,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,QAAsB;AAC1B,QAAI,KAAK,WAAW,UAAU,KAAK,MAAM,SAAS,GAAG;AACnD,cAAQ;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,WAAW,WAAW,6BAA6B;AAC/D,WAAO,KAAK,WAAW,aAAa,+BAA+B;AACnE,WAAO,KAAK,WAAW,SAAS,4BAA4B;AAE5D,UAAM,mBAAmB,KAAK,MAAM;AAAA,MAClC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,mBAAmB,GAAG;AAExB;AAAA,IACF;AAEA,SAAK,SAAS;AACd,QAAI,YAAY;AAChB,QAAI,wBAAwB;AAE5B,QAAI;AAEJ,WAAO,YAAY,KAAK,MAAM,QAAQ;AACpC,YAAM,OAAO,KAAK,MAAM,SAAS;AACjC;AAAA,QACE,KAAK,WAAW;AAAA,QAChB,2CAA2C,KAAK,MAAM;AAAA,MACxD;AACA,WAAK,SAAS;AAAA,QACZ,OAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI;AACF,aAAK,SAAS;AACd,YAAI;AACF,cAAI,KAAK,aAAa;AACpB,kBAAM,KAAK,YAAY,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ,MAAM,wBAAwB,CAAC;AAAA,QACzC;AACA;AAAA,UACE,CAAC,WAAW,UAAU,UAAU,EAAE,QAAQ,KAAK,IAAI,KAAK;AAAA,UACxD,0BAA0B,KAAK,IAAI;AAAA,QACrC;AAEA,cAAM,EAAE,UAAU,MAAM,IAAI;AAC5B,eAAO,UAAU,uCAAuC,KAAK,IAAI,EAAE;AAEnE,YAAI;AACJ,cAAM,kBAAmC;AAAA,UACvC;AAAA,UACA,SAAS,oBAAoB;AAAA,QAC/B;AAEA,YAAI,KAAK,SAAS,WAAW;AAC3B;AAAA,YACE,KAAK,YAAY,YACf,KAAK,YAAY,WACjB,KAAK,YAAY;AAAA,YACnB,gCAAgC,KAAK,OAAO;AAAA,UAC9C;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AACxD,cAAI,KAAK,YAAY,UAAU;AAC7B,iCACE,aACC;AAAA,UACL;AAAA,QACF,WAAW,KAAK,SAAS,YAAY,KAAK,SAAS,YAAY;AAC7D,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D,OAAO;AACL,kBAAQ;AAAA,YACN,0BAA0B,KAAK,IAAI;AAAA,UACrC;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D;AAEA,eAAO,OAAO,MAAM,WAAW;AAC/B,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD,aAAK,OAAO,SAAU,aAAqB,UAAU;AACrD;AAAA,MACF,SAAS,GAAQ;AACf,gCAAwB;AACxB,aAAK,QACH,GAAG,YAAY,OAAO,MAAM,WAAW,IAAI;AAC7C,aAAK,aAAa,EAAE;AAEpB,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF;AAAA,IACF;AAGA,aAAS,IAAI,YAAY,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AACtD,WAAK,MAAM,CAAC,EAAE,SAAS;AAAA,IACzB;AAEA,QAAI,uBAAuB;AACzB,WAAK,SAAS;AAAA,IAChB,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAEA,QAAI,KAAK,MAAM,QAAQ;AAErB,YAAM,cAAc,KAAK,IAAI,WAAW,KAAK,MAAM,SAAS,CAAC;AAC7D,aAAO,KAAK,MAAM,WAAW,EAAE;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,iBAA0B;AACxB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,kBAAwC;AACtC,QAAI,KAAK,WAAW,SAAS;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,iBAAiB,KAAK,MAAM;AAAA,MAChC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,kBAAkB,GAAG;AACvB,aAAO,KAAK,MAAM,cAAc;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAsB;AACpB,UAAM,WAA0B;AAAA,MAC9B,YAAY,WAAW;AAAA,MACvB,YAAY,YAAY,mBAAmB,KAAK;AAAA,MAChD,SAAS,KAAK,IAAI;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,IACd;AACA,WAAO;AAAA,EACT;AACF;;;ACtLA,SAAS,UAAAA,SAAQ,gBAAgB;;;ACPjC,SAAS,UAAAA,eAAc;AACvB,SAAS,YAAY;AAEd,SAAS,gBACd,MACA,gBACA;AACA,QAAM,SAAS,UAAU;AACzB,EAAAA,QAAO,QAAQ,+CAA+C;AAE9D,MAAI,mBAAmB;AACvB,MAAI,qBAAqB,wBAAwB,GAAG;AAClD,uBAAmB;AAAA,EACrB,WAAW,aAAa,GAAG;AACzB,uBAAmB,GAAG,aAAa,CAAC;AAAA,EACtC;AAEA,QAAM,WAAqB;AAAA,IACzB,YAAY,WAAW;AAAA,IACvB,SAAS,KAAK,IAAI;AAAA,IAClB,YAAY,YAAY,mBAAmB,KAAK;AAAA,IAChD,mBAAmB;AAAA,EACrB;AACA,QAAM,YAAyB;AAAA,IAC7B,OAAO,KAAK;AAAA,IACZ,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAEA,mBAAiB,SAAS;AAC5B;;;ADXA,IAAM,QAAQ,SAAS,YAAY;AACnC,IAAqB,UAArB,MAGE;AAAA,EAWA,YACE,SAGA,KACA;AAXF,sBAAoD;AAYlD,IAAAA,QAAO,SAAS,iCAAiC;AACjD,QAAI,OAAO,YAAY,YAAY;AACjC,WAAK,qBAAqB;AAAA,IAC5B,OAAO;AACL,WAAK,qBAAqB,MAAM,QAAQ,QAAQ,OAAO;AAAA,IACzD;AAEA,QAAI,OAAO,KAAK,eAAe,aAAa;AAC1C,WAAK,aAAa,IAAI;AAAA,IACxB;AACA,QAAI,OAAO,KAAK,aAAa,aAAa;AACxC,WAAK,WAAW,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,OAAoB,KAAyC;AACxE,UAAM,EAAE,OAAO,IAAI,OAAO,CAAC;AAC3B,UAAM,cAAc,OAAO,UAAU,WAAW,QAAQ,MAAM;AAC9D,IAAAA;AAAA,MACE,eAAe,KAAK;AAAA,MACpB;AAAA,IACF;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AACzB,QAAI,mBAAmB;AAEvB,QAAI,OAAO,UAAU,UAAU;AAC7B,yBAAmB,MAAM;AACzB,UAAI,CAAC,oBAAoB,MAAM,WAAW;AACxC,2BAAmB,MAAM;AAAA,MAC3B;AAAA,IACF;AAEA,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AAEtD,QAAI,aAA+B;AACnC,QAAI,wBAA4C;AAChD,QAAI,kBAA2C;AAC/C,QAAI,qBAEY;AAChB,QAAI,kBAAkB;AACpB,MAAAA;AAAA,QACE,aAAa;AAAA,QACb;AAAA,MACF;AAEA,2BAAqB,MAAM,gBAAgB;AAAA,QACzC;AAAA,QACA,oBAAoB;AAAA,MACtB,CAAC;AACD,MAAAA;AAAA,QACE,mBAAmB;AAAA,QACnB,gCAAgC,gBAAgB,IAC9C,mBAAmB,QAAQ,KAAK,mBAAmB,KAAK,KAAK,EAC/D;AAAA,MACF;AACA,8BAAwB,mBAAmB;AAC3C,wBAAkB,mBAAmB;AACrC,mBAAa,mBAAmB;AAAA,IAClC;AAEA,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,aAAa,aAAa,MAAM,IACzD,MAAM,gBAAgB;AAAA,MACpB,QAAQ,UAAU,KAAK;AAAA,MACvB;AAAA,MACA,0BAA0B;AAAA,MAC1B,aAAa,KAAK;AAAA,MAClB,cAAc;AAAA,IAChB,CAAC;AAEH,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,MACvC,gBAAgB,KAAK,UAAU,WAAW;AAAA,MAC1C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAiC,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAC3E;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT,SAAS;AAAA,MACX;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,gBAAgB,CAAC;AAAA,MACjB,aAAa;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,WAA0B,CAAC;AACjC,KAAC,YAAY,YAAY,CAAC,GAAG,QAAQ,CAAC,SAAS;AAC7C,UAAI,QAAQ,MAAM;AAChB,cAAM,UAAU,YAAY,KAAK,EAAE;AAEnC,YAAI,CAAC,SAAS;AACZ,kBAAQ;AAAA,YACN,kCAAkC,KAAK,EAAE;AAAA,UAC3C;AACA;AAAA,QACF;AACA,iBAAS,KAAK,OAAO;AAAA,MACvB;AAAA,IACF,CAAC;AAED;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,MACA;AAAA,IACF;AAEA,QAAI,UAAU;AACZ,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,IAAAA;AAAA,MACE,SAAS,UAAU;AAAA,MACnB,6CAA6C,SAAS,MAAM;AAAA,IAC9D;AAEA,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,UACP,IAAI,SAAS,CAAC,EAAG;AAAA,UACjB,SAAS,SAAS,CAAC,EAAG;AAAA,UACtB,QAAQ,SAAS,CAAC,EAAG;AAAA,UACrB,MAAM,SAAS,CAAC,EAAG;AAAA,QACrB;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA,EAQA,MAAM,QAAW,YAA+C;AAC9D,IAAAA;AAAA,MACE,OAAO,eAAe,YAAY,OAAO,eAAe;AAAA,MACxD,kDAAkD,OAAO,UAAU;AAAA,IACrE;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,SAAS;AAEvD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,IAAI,MAAM,qBAAwB;AAAA,MAC3D;AAAA,MACA,WAAW;AAAA,IACb,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,IACzC;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAwB,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAClE;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,EAAE,KAAK,IAAI,eAAe,CAAC;AAGjC;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,QAAI,YAAY,CAAC,MAAM;AACrB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,WAAsD;AACjE,QAAI,OAAO,cAAc,UAAU;AACjC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AACtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,aAAa,OAAO;AAAA,IAClD;AAEA,UAAM,EAAE,SAAS,KAAK,IAAI,aAAa;AACvC,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,eAAe;AAAA,MACf,kBAAkB;AAAA,MAClB,OAAO,OAAO,SAAY;AAAA,IAC5B;AACA,oBAAgB,UAAU,cAAc;AAExC,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AACF;;;AEhTA,IAAO,cAAQ","names":["assert"],"ignoreList":[],"sources":["../../src/ai-model/action-executor.ts","../../src/insight/index.ts","../../src/insight/utils.ts","../../src/index.ts"],"sourcesContent":["import { MIDSCENE_MODEL_NAME, getAIConfig } from '@/env';\nimport type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport { assert } from '@midscene/shared/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<any> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        task.timing.aiCost = (returnValue as any)?.aiCost || 0;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      return this.tasks[outputIndex].output;\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n","import { callAiFn, expandSearchArea } from '@/ai-model/common';\nimport { AiExtractElementInfo, AiLocateElement } from '@/ai-model/index';\nimport { AiAssert, AiLocateSection } from '@/ai-model/inspect';\nimport { vlLocateMode } from '@/env';\nimport type {\n  AIElementResponse,\n  AISingleElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DumpSubscriber,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  LocateParam,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { assert, getDebug } from '@midscene/shared/utils';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  callAI?: typeof callAiFn<AIElementResponse>;\n  quickAnswer?: Partial<AISingleElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(query: LocateParam, opt?: LocateOpts): Promise<LocateResult> {\n    const { callAI } = opt || {};\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(\n      queryPrompt || opt?.quickAnswer,\n      'query or quickAnswer is required for locate',\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n    let searchAreaPrompt = undefined;\n\n    if (typeof query === 'object') {\n      searchAreaPrompt = query.searchArea;\n      if (!searchAreaPrompt && query.deepThink) {\n        searchAreaPrompt = query.prompt;\n      }\n    }\n\n    const context = await this.contextRetrieverFn('locate');\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      assert(\n        vlLocateMode(),\n        'locate with search area is not supported with general purposed LLM. Please use Midscene VL model. https://midscenejs.com/choose-a-model',\n      );\n\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, elementById, rawResponse, usage } =\n      await AiLocateElement({\n        callAI: callAI || this.aiVendorFn,\n        context,\n        targetElementDescription: queryPrompt,\n        quickAnswer: opt?.quickAnswer,\n        searchConfig: searchAreaResponse,\n      });\n    // const parseResult = await this.aiVendorFn<AIElementParseResponse>(msgs);\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `locate - AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      context,\n      userQuery: {\n        element: queryPrompt,\n      },\n      quickAnswer: opt?.quickAnswer,\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T = any>(input: string): Promise<T>;\n  async extract<T extends Record<string, string>>(\n    input: T,\n  ): Promise<Record<keyof T, any>>;\n  async extract<T extends object>(input: Record<keyof T, string>): Promise<T>;\n\n  async extract<T>(dataDemand: InsightExtractParam): Promise<any> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      context,\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      usage,\n    };\n  }\n\n  async assert(assertion: string): Promise<InsightAssertionResponse> {\n    if (typeof assertion !== 'string') {\n      throw new Error(\n        'This is the assert method for Midscene, the first argument should be a string. If you want to use the assert method from Node.js, please import it from the Node.js assert module.',\n      );\n    }\n\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      context,\n      userQuery: {\n        assertion,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n}\n","import {\n  MIDSCENE_MODEL_NAME,\n  MIDSCENE_USE_VLM_UI_TARS,\n  getAIConfig,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@/env';\nimport type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getLogDir, getVersion, stringifyDumpData } from '@/utils';\nimport { assert } from '@midscene/shared/utils';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  dumpSubscriber?: DumpSubscriber,\n) {\n  const logDir = getLogDir();\n  assert(logDir, 'logDir should be set before writing dump file');\n\n  let modelDescription = '';\n  if (getAIConfigInBoolean(MIDSCENE_USE_VLM_UI_TARS)) {\n    modelDescription = 'vlm-ui-tars mode';\n  } else if (vlLocateMode()) {\n    modelDescription = `${vlLocateMode()} mode`;\n  }\n\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n    model_description: modelDescription,\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n}\n","import { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getLogDirByType, getVersion, setLogDir } from './utils';\n\nexport {\n  plan,\n  transformElementPositionToId,\n  describeUserPage,\n  AiLocateElement,\n  AiAssert,\n} from './ai-model/index';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from './env';\n\nexport type * from './types';\nexport default Insight;\nexport { Executor, setLogDir, getLogDirByType, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n} from './yaml';\n"]}
+{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAWA,SAAS,cAAc;AAEhB,IAAM,WAAN,MAAe;AAAA,EAUpB,YACE,MACA,SAGA;AACA,SAAK,SACH,SAAS,SAAS,QAAQ,MAAM,SAAS,IAAI,YAAY;AAC3D,SAAK,OAAO;AACZ,SAAK,SAAS,SAAS,SAAS,CAAC,GAAG;AAAA,MAAI,CAAC,SACvC,KAAK,kBAAkB,IAAI;AAAA,IAC7B;AACA,SAAK,cAAc,SAAS;AAAA,EAC9B;AAAA,EAEQ,kBAAkB,MAAyC;AACjE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAgE;AAC3E;AAAA,MACE,KAAK,WAAW;AAAA,MAChB;AAAA,QAAyD,KAAK,gBAAgB,GAAG,KAAK;AAAA,EAAK,KAAK,gBAAgB,GAAG,UAAU;AAAA,IAC/H;AACA,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAK,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,SAAS,KAAK,kBAAkB,IAAI,CAAC,CAAC;AAAA,IACrE,OAAO;AACL,WAAK,MAAM,KAAK,KAAK,kBAAkB,IAAI,CAAC;AAAA,IAC9C;AACA,QAAI,KAAK,WAAW,WAAW;AAC7B,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,QAAsB;AAC1B,QAAI,KAAK,WAAW,UAAU,KAAK,MAAM,SAAS,GAAG;AACnD,cAAQ;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,WAAW,WAAW,6BAA6B;AAC/D,WAAO,KAAK,WAAW,aAAa,+BAA+B;AACnE,WAAO,KAAK,WAAW,SAAS,4BAA4B;AAE5D,UAAM,mBAAmB,KAAK,MAAM;AAAA,MAClC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,mBAAmB,GAAG;AAExB;AAAA,IACF;AAEA,SAAK,SAAS;AACd,QAAI,YAAY;AAChB,QAAI,wBAAwB;AAE5B,QAAI;AAEJ,WAAO,YAAY,KAAK,MAAM,QAAQ;AACpC,YAAM,OAAO,KAAK,MAAM,SAAS;AACjC;AAAA,QACE,KAAK,WAAW;AAAA,QAChB,2CAA2C,KAAK,MAAM;AAAA,MACxD;AACA,WAAK,SAAS;AAAA,QACZ,OAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI;AACF,aAAK,SAAS;AACd,YAAI;AACF,cAAI,KAAK,aAAa;AACpB,kBAAM,KAAK,YAAY,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ,MAAM,wBAAwB,CAAC;AAAA,QACzC;AACA;AAAA,UACE,CAAC,WAAW,UAAU,UAAU,EAAE,QAAQ,KAAK,IAAI,KAAK;AAAA,UACxD,0BAA0B,KAAK,IAAI;AAAA,QACrC;AAEA,cAAM,EAAE,UAAU,MAAM,IAAI;AAC5B,eAAO,UAAU,uCAAuC,KAAK,IAAI,EAAE;AAEnE,YAAI;AACJ,cAAM,kBAAmC;AAAA,UACvC;AAAA,UACA,SAAS,oBAAoB;AAAA,QAC/B;AAEA,YAAI,KAAK,SAAS,WAAW;AAC3B;AAAA,YACE,KAAK,YAAY,YACf,KAAK,YAAY,WACjB,KAAK,YAAY;AAAA,YACnB,gCAAgC,KAAK,OAAO;AAAA,UAC9C;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AACxD,cAAI,KAAK,YAAY,UAAU;AAC7B,iCACE,aACC;AAAA,UACL;AAAA,QACF,WAAW,KAAK,SAAS,YAAY,KAAK,SAAS,YAAY;AAC7D,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D,OAAO;AACL,kBAAQ;AAAA,YACN,0BAA0B,KAAK,IAAI;AAAA,UACrC;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D;AAEA,eAAO,OAAO,MAAM,WAAW;AAC/B,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD,aAAK,OAAO,SAAU,aAAqB,UAAU;AACrD;AAAA,MACF,SAAS,GAAQ;AACf,gCAAwB;AACxB,aAAK,QACH,GAAG,YAAY,OAAO,MAAM,WAAW,IAAI;AAC7C,aAAK,aAAa,EAAE;AAEpB,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF;AAAA,IACF;AAGA,aAAS,IAAI,YAAY,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AACtD,WAAK,MAAM,CAAC,EAAE,SAAS;AAAA,IACzB;AAEA,QAAI,uBAAuB;AACzB,WAAK,SAAS;AAAA,IAChB,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAEA,QAAI,KAAK,MAAM,QAAQ;AAErB,YAAM,cAAc,KAAK,IAAI,WAAW,KAAK,MAAM,SAAS,CAAC;AAC7D,aAAO,KAAK,MAAM,WAAW,EAAE;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,iBAA0B;AACxB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,kBAAwC;AACtC,QAAI,KAAK,WAAW,SAAS;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,iBAAiB,KAAK,MAAM;AAAA,MAChC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,kBAAkB,GAAG;AACvB,aAAO,KAAK,MAAM,cAAc;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAsB;AACpB,UAAM,WAA0B;AAAA,MAC9B,YAAY,WAAW;AAAA,MACvB,YAAY,YAAY,mBAAmB,KAAK;AAAA,MAChD,SAAS,KAAK,IAAI;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,IACd;AACA,WAAO;AAAA,EACT;AACF;;;AClLA,SAAS,gBAAgB;AACzB,SAAS,UAAAA,eAAc;;;ACZvB,SAAS,UAAAA,eAAc;AACvB,SAAS,YAAY;AAEd,SAAS,gBACd,MACA,gBACA;AACA,QAAM,SAAS,UAAU;AACzB,EAAAA,QAAO,QAAQ,+CAA+C;AAE9D,MAAI,mBAAmB;AACvB,MAAI,qBAAqB,wBAAwB,GAAG;AAClD,uBAAmB;AAAA,EACrB,WAAW,aAAa,GAAG;AACzB,uBAAmB,GAAG,aAAa,CAAC;AAAA,EACtC;AAEA,QAAM,WAAqB;AAAA,IACzB,YAAY,WAAW;AAAA,IACvB,SAAS,KAAK,IAAI;AAAA,IAClB,YAAY,YAAY,mBAAmB,KAAK;AAAA,IAChD,mBAAmB;AAAA,EACrB;AACA,QAAM,YAAyB;AAAA,IAC7B,OAAO,KAAK;AAAA,IACZ,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAEA,mBAAiB,SAAS;AAC5B;;;ADNA,IAAM,QAAQ,SAAS,YAAY;AACnC,IAAqB,UAArB,MAGE;AAAA,EAWA,YACE,SAGA,KACA;AAXF,sBAAoD;AAYlD,IAAAA,QAAO,SAAS,iCAAiC;AACjD,QAAI,OAAO,YAAY,YAAY;AACjC,WAAK,qBAAqB;AAAA,IAC5B,OAAO;AACL,WAAK,qBAAqB,MAAM,QAAQ,QAAQ,OAAO;AAAA,IACzD;AAEA,QAAI,OAAO,KAAK,eAAe,aAAa;AAC1C,WAAK,aAAa,IAAI;AAAA,IACxB;AACA,QAAI,OAAO,KAAK,aAAa,aAAa;AACxC,WAAK,WAAW,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,OACJ,OACA,KACuB;AACvB,UAAM,EAAE,OAAO,IAAI,OAAO,CAAC;AAC3B,UAAM,cAAc,OAAO,UAAU,WAAW,QAAQ,MAAM;AAC9D,IAAAA;AAAA,MACE,eAAe,KAAK;AAAA,MACpB;AAAA,IACF;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AACzB,QAAI,mBAAmB;AAEvB,IAAAA,QAAO,OAAO,UAAU,UAAU,sCAAsC;AACxE,uBAAmB,MAAM;AAEzB,UAAM,wBAAwB;AAAA,MAC5B;AAAA,IACF;AACA,QAAI,uBAAuB;AACzB,YAAM,yBAAyB,qBAAqB;AAAA,IACtD;AACA,QAAI,CAAC,qBAAqB,MAAM,aAAa,wBAAwB;AACnE,yBAAmB,MAAM;AAAA,IAC3B;AAEA,QAAI,oBAAoB,CAAC,aAAa,GAAG;AACvC,cAAQ;AAAA,QACN;AAAA,MACF;AACA,yBAAmB;AAAA,IACrB;AAEA,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AAEtD,QAAI,aAA+B;AACnC,QAAI,wBAA4C;AAChD,QAAI,kBAA2C;AAC/C,QAAI,qBAEY;AAChB,QAAI,kBAAkB;AACpB,2BAAqB,MAAM,gBAAgB;AAAA,QACzC;AAAA,QACA,oBAAoB;AAAA,MACtB,CAAC;AACD,MAAAA;AAAA,QACE,mBAAmB;AAAA,QACnB,gCAAgC,gBAAgB,IAC9C,mBAAmB,QAAQ,KAAK,mBAAmB,KAAK,KAAK,EAC/D;AAAA,MACF;AACA,8BAAwB,mBAAmB;AAC3C,wBAAkB,mBAAmB;AACrC,mBAAa,mBAAmB;AAAA,IAClC;AAEA,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,aAAa,aAAa,MAAM,IACzD,MAAM,gBAAgB;AAAA,MACpB,QAAQ,UAAU,KAAK;AAAA,MACvB;AAAA,MACA,0BAA0B;AAAA,MAC1B,aAAa,KAAK;AAAA,MAClB,cAAc;AAAA,IAChB,CAAC;AAEH,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,MACvC,gBAAgB,KAAK,UAAU,WAAW;AAAA,MAC1C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAiC,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAC3E;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT,SAAS;AAAA,MACX;AAAA,MACA,aAAa,KAAK;AAAA,MAClB,gBAAgB,CAAC;AAAA,MACjB,aAAa;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,WAAW,CAAC,CAAC;AAAA,MACb,OAAO;AAAA,IACT;AAEA,UAAM,WAA0B,CAAC;AACjC,KAAC,YAAY,YAAY,CAAC,GAAG,QAAQ,CAAC,SAAS;AAC7C,UAAI,QAAQ,MAAM;AAChB,cAAM,UAAU,YAAY,KAAK,EAAE;AAEnC,YAAI,CAAC,SAAS;AACZ,kBAAQ;AAAA,YACN,kCAAkC,KAAK,EAAE;AAAA,UAC3C;AACA;AAAA,QACF;AACA,iBAAS,KAAK,OAAO;AAAA,MACvB;AAAA,IACF,CAAC;AAED;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,MACA;AAAA,IACF;AAEA,QAAI,UAAU;AACZ,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,IAAAA;AAAA,MACE,SAAS,UAAU;AAAA,MACnB,6CAA6C,SAAS,MAAM;AAAA,IAC9D;AAEA,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,UACP,IAAI,SAAS,CAAC,EAAG;AAAA,UACjB,SAAS,SAAS,CAAC,EAAG;AAAA,UACtB,QAAQ,SAAS,CAAC,EAAG;AAAA,UACrB,MAAM,SAAS,CAAC,EAAG;AAAA,QACrB;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA,EAQA,MAAM,QAAW,YAA+C;AAC9D,IAAAA;AAAA,MACE,OAAO,eAAe,YAAY,OAAO,eAAe;AAAA,MACxD,kDAAkD,OAAO,UAAU;AAAA,IACrE;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,SAAS;AAEvD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,IAAI,MAAM,qBAAwB;AAAA,MAC3D;AAAA,MACA,WAAW;AAAA,IACb,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,IACzC;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAwB,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAClE;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,EAAE,KAAK,IAAI,eAAe,CAAC;AAGjC;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,QAAI,YAAY,CAAC,MAAM;AACrB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,WAAsD;AACjE,QAAI,OAAO,cAAc,UAAU;AACjC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AACtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,aAAa,OAAO;AAAA,IAClD;AAEA,UAAM,EAAE,SAAS,KAAK,IAAI,aAAa;AACvC,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,eAAe;AAAA,MACf,kBAAkB;AAAA,MAClB,OAAO,OAAO,SAAY;AAAA,IAC5B;AACA,oBAAgB,UAAU,cAAc;AAExC,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AACF;;;AEjUA,IAAO,cAAQ","names":["assert"],"ignoreList":[],"sources":["../../src/ai-model/action-executor.ts","../../src/insight/index.ts","../../src/insight/utils.ts","../../src/index.ts"],"sourcesContent":["import { MIDSCENE_MODEL_NAME, getAIConfig } from '@/env';\nimport type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport { assert } from '@midscene/shared/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<any> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        task.timing.aiCost = (returnValue as any)?.aiCost || 0;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      return this.tasks[outputIndex].output;\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n","import { callAiFn, expandSearchArea } from '@/ai-model/common';\nimport { AiExtractElementInfo, AiLocateElement } from '@/ai-model/index';\nimport { AiAssert, AiLocateSection } from '@/ai-model/inspect';\nimport {\n  MIDSCENE_FORCE_DEEP_THINK,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@/env';\nimport type {\n  AIElementResponse,\n  AISingleElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  callAI?: typeof callAiFn<AIElementResponse>;\n  quickAnswer?: Partial<AISingleElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt?: LocateOpts,\n  ): Promise<LocateResult> {\n    const { callAI } = opt || {};\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(\n      queryPrompt || opt?.quickAnswer,\n      'query or quickAnswer is required for locate',\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n    let searchAreaPrompt = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n    searchAreaPrompt = query.searchArea;\n\n    const globalDeepThinkSwitch = getAIConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    if (!searchAreaPrompt && (query.deepThink || globalDeepThinkSwitch)) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    if (searchAreaPrompt && !vlLocateMode()) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with general purposed LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = await this.contextRetrieverFn('locate');\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, elementById, rawResponse, usage } =\n      await AiLocateElement({\n        callAI: callAI || this.aiVendorFn,\n        context,\n        targetElementDescription: queryPrompt,\n        quickAnswer: opt?.quickAnswer,\n        searchConfig: searchAreaResponse,\n      });\n    // const parseResult = await this.aiVendorFn<AIElementParseResponse>(msgs);\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `locate - AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      context,\n      userQuery: {\n        element: queryPrompt,\n      },\n      quickAnswer: opt?.quickAnswer,\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T = any>(input: string): Promise<T>;\n  async extract<T extends Record<string, string>>(\n    input: T,\n  ): Promise<Record<keyof T, any>>;\n  async extract<T extends object>(input: Record<keyof T, string>): Promise<T>;\n\n  async extract<T>(dataDemand: InsightExtractParam): Promise<any> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      context,\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      usage,\n    };\n  }\n\n  async assert(assertion: string): Promise<InsightAssertionResponse> {\n    if (typeof assertion !== 'string') {\n      throw new Error(\n        'This is the assert method for Midscene, the first argument should be a string. If you want to use the assert method from Node.js, please import it from the Node.js assert module.',\n      );\n    }\n\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      context,\n      userQuery: {\n        assertion,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n}\n","import {\n  MIDSCENE_MODEL_NAME,\n  MIDSCENE_USE_VLM_UI_TARS,\n  getAIConfig,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@/env';\nimport type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getLogDir, getVersion, stringifyDumpData } from '@/utils';\nimport { assert } from '@midscene/shared/utils';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  dumpSubscriber?: DumpSubscriber,\n) {\n  const logDir = getLogDir();\n  assert(logDir, 'logDir should be set before writing dump file');\n\n  let modelDescription = '';\n  if (getAIConfigInBoolean(MIDSCENE_USE_VLM_UI_TARS)) {\n    modelDescription = 'vlm-ui-tars mode';\n  } else if (vlLocateMode()) {\n    modelDescription = `${vlLocateMode()} mode`;\n  }\n\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n    model_description: modelDescription,\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n}\n","import { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getLogDirByType, getVersion, setLogDir } from './utils';\n\nexport {\n  plan,\n  transformElementPositionToId,\n  describeUserPage,\n  AiLocateElement,\n  AiAssert,\n} from './ai-model/index';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from './env';\n\nexport type * from './types';\nexport default Insight;\nexport { Executor, setLogDir, getLogDirByType, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n} from './yaml';\n"]}

package/dist/es/{llm-planning-ef8544fb.d.ts → llm-planning-44e0084c.d.ts}

@@ -1,4 +1,4 @@
-import { k as AIUsageInfo, R as Rect, B as BaseElement, U as UIContext, q as AIElementResponse, j as ElementTreeNode, o as AIElementLocatorResponse, A as AISingleElementResponse, n as AISingleElementResponseByPosition, H as ElementById, r as AIDataExtractionResponse, t as AIAssertionResponse, T as PlanningAIResponse } from './types-f9d27111.js';
+import { k as AIUsageInfo, R as Rect, B as BaseElement, U as UIContext, q as AIElementResponse, j as ElementTreeNode, o as AIElementLocatorResponse, A as AISingleElementResponse, n as AISingleElementResponseByPosition, H as ElementById, r as AIDataExtractionResponse, t as AIAssertionResponse, T as PlanningAIResponse } from './types-9eeb8ca7.js';
 import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
 
 type AIArgs = [
@@ -43,7 +43,7 @@ declare function describeUserPage<ElementType extends BaseElement = BaseElement>
 declare function transformElementPositionToId(aiResult: AIElementResponse | [number, number], treeRoot: ElementTreeNode<BaseElement>, size: {
     width: number;
     height: number;
-}, insertElementByPosition: (position: {
+}, searchAreaRect: Rect | undefined, insertElementByPosition: (position: {
     x: number;
     y: number;
 }) => BaseElement): Promise<AIElementLocatorResponse>;

package/dist/es/tree.d.ts

@@ -1,5 +1,5 @@
 import * as _midscene_shared_constants from '@midscene/shared/constants';
-import { B as BaseElement, j as ElementTreeNode } from './types-f9d27111.js';
+import { B as BaseElement, j as ElementTreeNode } from './types-9eeb8ca7.js';
 import 'openai/resources';
 
 declare function truncateText(text: string | number | object | undefined, maxLength?: number): string;

package/dist/{types/types-f9d27111.d.ts → es/types-9eeb8ca7.d.ts}

@@ -1,12 +1,13 @@
 import { NodeType } from '@midscene/shared/constants';
 import { ChatCompletionMessageParam } from 'openai/resources';
 
-interface DetailedLocateParam {
-    prompt: string;
+interface LocateOption {
     searchArea?: string;
     deepThink?: boolean;
 }
-type LocateParam = string | DetailedLocateParam;
+interface DetailedLocateParam extends LocateOption {
+    prompt: string;
+}
 interface scrollParam {
     direction: 'down' | 'up' | 'right' | 'left';
     scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
@@ -54,23 +55,23 @@ interface MidsceneYamlFlowItemAIWaitFor {
     aiWaitFor: string;
     timeout?: number;
 }
-interface MidsceneYamlFlowItemAITap {
-    aiTap: LocateParam;
+interface MidsceneYamlFlowItemAITap extends LocateOption {
+    aiTap: string;
 }
-interface MidsceneYamlFlowItemAIHover {
-    aiHover: LocateParam;
+interface MidsceneYamlFlowItemAIHover extends LocateOption {
+    aiHover: string;
 }
-interface MidsceneYamlFlowItemAIInput {
+interface MidsceneYamlFlowItemAIInput extends LocateOption {
     aiInput: string;
-    locate: LocateParam;
+    locate: string;
 }
-interface MidsceneYamlFlowItemAIKeyboardPress {
+interface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {
     aiKeyboardPress: string;
-    locate?: LocateParam;
+    locate?: string;
 }
-interface MidsceneYamlFlowItemAIScroll {
-    aiScroll: PlanningActionParamScroll;
-    locate?: LocateParam;
+interface MidsceneYamlFlowItemAIScroll extends LocateOption, PlanningActionParamScroll {
+    aiScroll: null;
+    locate?: string;
 }
 interface MidsceneYamlFlowItemSleep {
     sleep: number;
@@ -234,6 +235,7 @@ interface InsightDump extends DumpMeta {
     quickAnswer?: Partial<AISingleElementResponse> | null;
     matchedElement: BaseElement[];
     matchedRect?: Rect;
+    deepThink?: boolean;
     data: any;
     assertionPass?: boolean;
     assertionThought?: string;
@@ -275,7 +277,7 @@ interface PlanningAction<ParamType = any> {
     thought?: string;
     type: 'Locate' | 'Tap' | 'Drag' | 'Hover' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Error' | 'ExpectedFalsyCondition' | 'Assert' | 'AssertWithoutThrow' | 'Sleep' | 'Finished';
     param: ParamType;
-    locate: PlanningLocateParam | null;
+    locate?: PlanningLocateParam | null;
 }
 interface PlanningAIResponse {
     action?: PlanningAction;
@@ -342,7 +344,7 @@ interface ExecutionTaskApply<Type extends ExecutionTaskType = any, TaskParam = a
     subType?: string;
     param?: TaskParam;
     thought?: string;
-    locate: PlanningLocateParam | null;
+    locate?: PlanningLocateParam | null;
     quickAnswer?: AISingleElementResponse | null;
     pageContext?: UIContext;
     executor: (param: TaskParam, context: ExecutorContext) => Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void> | undefined | void;
@@ -405,4 +407,4 @@ interface GroupedActionDump {
     executions: ExecutionDump[];
 }
 
-export { type PlanningActionParamError as $, type AISingleElementResponse as A, BaseElement as B, type CallAIFn as C, type DumpSubscriber as D, type ExecutionTask as E, type PartialInsightDumpFromSDK as F, type LiteUISection as G, type ElementById as H, type InsightAction as I, type AgentWaitForOpt as J, type AgentAssertOpt as K, type LocateParam as L, type MidsceneYamlScript as M, type PlanningLocateParam as N, type OnTaskStartTip as O, type Point as P, type PlanningAction as Q, type Rect as R, type Size as S, type PlanningAIResponse as T, UIContext as U, type PlanningActionParamTap as V, type PlanningActionParamHover as W, type PlanningActionParamInputOrKeyPress as X, type PlanningActionParamScroll as Y, type PlanningActionParamAssert as Z, type PlanningActionParamSleep as _, type ExecutionTaskProgressOptions as a, type PlanningActionParamWaitFor as a0, type Color as a1, type BaseAgentParserOpt as a2, type PuppeteerParserOpt as a3, type PlaywrightParserOpt as a4, type ExecutionRecorderItem as a5, type ExecutionTaskType as a6, type ExecutorContext as a7, type TaskCacheInfo as a8, type ExecutionTaskReturn as a9, type MidsceneYamlFlowItemAIInput as aA, type MidsceneYamlFlowItemAIKeyboardPress as aB, type MidsceneYamlFlowItemAIScroll as aC, type MidsceneYamlFlowItemSleep as aD, type FreeFn as aE, type ScriptPlayerTaskStatus as aF, type ScriptPlayerStatusValue as aG, type ExecutionTaskInsightLocateParam as aa, type ExecutionTaskInsightLocateOutput as ab, type ExecutionTaskInsightDumpLog as ac, type ExecutionTaskInsightLocateApply as ad, type ExecutionTaskInsightLocate as ae, type ExecutionTaskInsightQueryParam as af, type ExecutionTaskInsightQueryOutput as ag, type ExecutionTaskInsightQueryApply as ah, type ExecutionTaskInsightQuery as ai, type ExecutionTaskInsightAssertionParam as aj, type ExecutionTaskInsightAssertionApply as ak, type ExecutionTaskInsightAssertion as al, type ExecutionTaskActionApply as am, type ExecutionTaskAction as an, type ExecutionTaskPlanningApply as ao, type ExecutionTaskPlanning as ap, type GroupedActionDump as aq, type DetailedLocateParam as ar, type scrollParam as as, type MidsceneYamlScriptEnv as at, type MidsceneYamlFlowItemAIAction as au, type MidsceneYamlFlowItemAIAssert as av, type MidsceneYamlFlowItemAIQuery as aw, type MidsceneYamlFlowItemAIWaitFor as ax, type MidsceneYamlFlowItemAITap as ay, type MidsceneYamlFlowItemAIHover as az, type ExecutionTaskApply as b, type ExecutionDump as c, type InsightTaskInfo as d, type InsightOptions as e, type LocateResult as f, type InsightAssertionResponse as g, type MidsceneYamlTask as h, type MidsceneYamlFlowItem as i, type ElementTreeNode as j, type AIUsageInfo as k, AIResponseFormat as l, type AISingleElementResponseById as m, type AISingleElementResponseByPosition as n, type AIElementLocatorResponse as o, type AIElementCoordinatesResponse as p, type AIElementResponse as q, type AIDataExtractionResponse as r, type AISectionLocatorResponse as s, type AIAssertionResponse as t, type EnsureObject as u, type InsightExtractParam as v, type LocateResultElement as w, type DumpMeta as x, type ReportDumpWithAttributes as y, type InsightDump as z };
+export { type PlanningActionParamError as $, type AISingleElementResponse as A, BaseElement as B, type CallAIFn as C, type DumpSubscriber as D, type ExecutionTask as E, type PartialInsightDumpFromSDK as F, type LiteUISection as G, type ElementById as H, type InsightAction as I, type AgentWaitForOpt as J, type AgentAssertOpt as K, type LocateResult as L, type MidsceneYamlScript as M, type PlanningLocateParam as N, type OnTaskStartTip as O, type Point as P, type PlanningAction as Q, type Rect as R, type Size as S, type PlanningAIResponse as T, UIContext as U, type PlanningActionParamTap as V, type PlanningActionParamHover as W, type PlanningActionParamInputOrKeyPress as X, type PlanningActionParamScroll as Y, type PlanningActionParamAssert as Z, type PlanningActionParamSleep as _, type ExecutionTaskProgressOptions as a, type PlanningActionParamWaitFor as a0, type Color as a1, type BaseAgentParserOpt as a2, type PuppeteerParserOpt as a3, type PlaywrightParserOpt as a4, type ExecutionRecorderItem as a5, type ExecutionTaskType as a6, type ExecutorContext as a7, type TaskCacheInfo as a8, type ExecutionTaskReturn as a9, type MidsceneYamlFlowItemAIInput as aA, type MidsceneYamlFlowItemAIKeyboardPress as aB, type MidsceneYamlFlowItemAIScroll as aC, type MidsceneYamlFlowItemSleep as aD, type FreeFn as aE, type ScriptPlayerTaskStatus as aF, type ScriptPlayerStatusValue as aG, type ExecutionTaskInsightLocateParam as aa, type ExecutionTaskInsightLocateOutput as ab, type ExecutionTaskInsightDumpLog as ac, type ExecutionTaskInsightLocateApply as ad, type ExecutionTaskInsightLocate as ae, type ExecutionTaskInsightQueryParam as af, type ExecutionTaskInsightQueryOutput as ag, type ExecutionTaskInsightQueryApply as ah, type ExecutionTaskInsightQuery as ai, type ExecutionTaskInsightAssertionParam as aj, type ExecutionTaskInsightAssertionApply as ak, type ExecutionTaskInsightAssertion as al, type ExecutionTaskActionApply as am, type ExecutionTaskAction as an, type ExecutionTaskPlanningApply as ao, type ExecutionTaskPlanning as ap, type GroupedActionDump as aq, type LocateOption as ar, type scrollParam as as, type MidsceneYamlScriptEnv as at, type MidsceneYamlFlowItemAIAction as au, type MidsceneYamlFlowItemAIAssert as av, type MidsceneYamlFlowItemAIQuery as aw, type MidsceneYamlFlowItemAIWaitFor as ax, type MidsceneYamlFlowItemAITap as ay, type MidsceneYamlFlowItemAIHover as az, type ExecutionTaskApply as b, type ExecutionDump as c, type InsightTaskInfo as d, type InsightOptions as e, type DetailedLocateParam as f, type InsightAssertionResponse as g, type MidsceneYamlTask as h, type MidsceneYamlFlowItem as i, type ElementTreeNode as j, type AIUsageInfo as k, AIResponseFormat as l, type AISingleElementResponseById as m, type AISingleElementResponseByPosition as n, type AIElementLocatorResponse as o, type AIElementCoordinatesResponse as p, type AIElementResponse as q, type AIDataExtractionResponse as r, type AISectionLocatorResponse as s, type AIAssertionResponse as t, type EnsureObject as u, type InsightExtractParam as v, type LocateResultElement as w, type DumpMeta as x, type ReportDumpWithAttributes as y, type InsightDump as z };

package/dist/es/utils.d.ts

@@ -1,4 +1,4 @@
-import { y as ReportDumpWithAttributes, R as Rect } from './types-f9d27111.js';
+import { y as ReportDumpWithAttributes, R as Rect } from './types-9eeb8ca7.js';
 import '@midscene/shared/constants';
 import 'openai/resources';
 

package/dist/es/utils.js

@@ -16,8 +16,8 @@ import {
   uploadTestInfoToServer,
   writeDumpReport,
   writeLogFile
-} from "./chunk-QKOUPDNQ.js";
-import "./chunk-I2I7OU2E.js";
+} from "./chunk-JYD7DE4D.js";
+import "./chunk-UBGEKXK7.js";
 export {
   getLogDir,
   getLogDirByType,

package/dist/lib/ai-model.d.ts

@@ -1,8 +1,8 @@
-import { k as AIUsageInfo, Q as PlanningAction } from './types-f9d27111.js';
+import { k as AIUsageInfo, Q as PlanningAction } from './types-9eeb8ca7.js';
 import { ChatCompletionMessageParam } from 'openai/resources';
 export { ChatCompletionMessageParam } from 'openai/resources';
-import { b as AIActionType } from './llm-planning-ef8544fb.js';
-export { a as AiAssert, e as AiExtractElementInfo, A as AiLocateElement, f as AiLocateSection, g as adaptBboxToRect, c as callAiFn, d as describeUserPage, p as plan, t as transformElementPositionToId } from './llm-planning-ef8544fb.js';
+import { b as AIActionType } from './llm-planning-44e0084c.js';
+export { a as AiAssert, e as AiExtractElementInfo, A as AiLocateElement, f as AiLocateSection, g as adaptBboxToRect, c as callAiFn, d as describeUserPage, p as plan, t as transformElementPositionToId } from './llm-planning-44e0084c.js';
 import { actionParser } from '@ui-tars/action-parser';
 import '@midscene/shared/constants';
 

package/dist/lib/ai-model.js

@@ -11,8 +11,8 @@
 
 
 
-var _chunkKUTJMPAQjs = require('./chunk-KUTJMPAQ.js');
-require('./chunk-I2I7OU2E.js');
+var _chunkZSMRRNLVjs = require('./chunk-ZSMRRNLV.js');
+require('./chunk-UBGEKXK7.js');
 
 
 
@@ -26,4 +26,4 @@ require('./chunk-I2I7OU2E.js');
 
 
 
-exports.AiAssert = _chunkKUTJMPAQjs.AiAssert; exports.AiExtractElementInfo = _chunkKUTJMPAQjs.AiExtractElementInfo; exports.AiLocateElement = _chunkKUTJMPAQjs.AiLocateElement; exports.AiLocateSection = _chunkKUTJMPAQjs.AiLocateSection; exports.adaptBboxToRect = _chunkKUTJMPAQjs.adaptBboxToRect; exports.callAiFn = _chunkKUTJMPAQjs.callAiFn; exports.callToGetJSONObject = _chunkKUTJMPAQjs.callToGetJSONObject; exports.describeUserPage = _chunkKUTJMPAQjs.describeUserPage; exports.plan = _chunkKUTJMPAQjs.plan; exports.systemPromptToLocateElement = _chunkKUTJMPAQjs.systemPromptToLocateElement; exports.transformElementPositionToId = _chunkKUTJMPAQjs.transformElementPositionToId; exports.vlmPlanning = _chunkKUTJMPAQjs.vlmPlanning;
+exports.AiAssert = _chunkZSMRRNLVjs.AiAssert; exports.AiExtractElementInfo = _chunkZSMRRNLVjs.AiExtractElementInfo; exports.AiLocateElement = _chunkZSMRRNLVjs.AiLocateElement; exports.AiLocateSection = _chunkZSMRRNLVjs.AiLocateSection; exports.adaptBboxToRect = _chunkZSMRRNLVjs.adaptBboxToRect; exports.callAiFn = _chunkZSMRRNLVjs.callAiFn; exports.callToGetJSONObject = _chunkZSMRRNLVjs.callToGetJSONObject; exports.describeUserPage = _chunkZSMRRNLVjs.describeUserPage; exports.plan = _chunkZSMRRNLVjs.plan; exports.systemPromptToLocateElement = _chunkZSMRRNLVjs.systemPromptToLocateElement; exports.transformElementPositionToId = _chunkZSMRRNLVjs.transformElementPositionToId; exports.vlmPlanning = _chunkZSMRRNLVjs.vlmPlanning;

package/dist/lib/{chunk-QKOUPDNQ.js → chunk-JYD7DE4D.js}

@@ -3,7 +3,7 @@
 
 
 
-var _chunkI2I7OU2Ejs = require('./chunk-I2I7OU2E.js');
+var _chunkUBGEKXK7js = require('./chunk-UBGEKXK7.js');
 
 // src/utils.ts
 var _child_process = require('child_process');
@@ -196,10 +196,10 @@ function stringifyDumpData(data, indents) {
   return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-  return "0.13.1";
+  return "0.13.2-beta-20250401015137.0";
 }
 function debugLog(...message) {
-  const debugMode = _chunkI2I7OU2Ejs.getAIConfig.call(void 0, _chunkI2I7OU2Ejs.MIDSCENE_DEBUG_MODE);
+  const debugMode = _chunkUBGEKXK7js.getAIConfig.call(void 0, _chunkUBGEKXK7js.MIDSCENE_DEBUG_MODE);
   if (debugMode) {
     console.log("[Midscene]", ...message);
   }
@@ -208,7 +208,7 @@ var lastReportedRepoUrl = "";
 function uploadTestInfoToServer({ testUrl }) {
   let repoUrl = "";
   let userEmail = "";
-  const extraConfig = _chunkI2I7OU2Ejs.getAIConfigInJson.call(void 0, _chunkI2I7OU2Ejs.MIDSCENE_OPENAI_INIT_CONFIG_JSON);
+  const extraConfig = _chunkUBGEKXK7js.getAIConfigInJson.call(void 0, _chunkUBGEKXK7js.MIDSCENE_OPENAI_INIT_CONFIG_JSON);
   const serverUrl = _optionalChain([extraConfig, 'optionalAccess', _6 => _6.REPORT_SERVER_URL]);
   try {
     repoUrl = _child_process.execSync.call(void 0, "git config --get remote.origin.url").toString().trim();
@@ -262,4 +262,4 @@ function uploadTestInfoToServer({ testUrl }) {
 
 exports.groupedActionDumpFileExt = groupedActionDumpFileExt; exports.getLogDir = getLogDir; exports.setLogDir = setLogDir; exports.getLogDirByType = getLogDirByType; exports.setReportTpl = setReportTpl; exports.replaceStringWithFirstAppearance = replaceStringWithFirstAppearance; exports.reportHTMLContent = reportHTMLContent; exports.writeDumpReport = writeDumpReport; exports.writeLogFile = writeLogFile; exports.getTmpDir = getTmpDir; exports.getTmpFile = getTmpFile; exports.overlapped = overlapped; exports.sleep = sleep; exports.replacerForPageObject = replacerForPageObject; exports.stringifyDumpData = stringifyDumpData; exports.getVersion = getVersion; exports.uploadTestInfoToServer = uploadTestInfoToServer;
 
-//# sourceMappingURL=chunk-QKOUPDNQ.js.map
+//# sourceMappingURL=chunk-JYD7DE4D.js.map

package/dist/lib/{chunk-I2I7OU2E.js → chunk-UBGEKXK7.js}

@@ -6,6 +6,7 @@ var MIDSCENE_DEBUG_AI_PROFILE = "MIDSCENE_DEBUG_AI_PROFILE";
 var MIDSCENE_DEBUG_AI_RESPONSE = "MIDSCENE_DEBUG_AI_RESPONSE";
 var MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG = "MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG";
 var MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
+var MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
 var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
 var OPENAI_API_KEY = "OPENAI_API_KEY";
 var OPENAI_BASE_URL = "OPENAI_BASE_URL";
@@ -33,6 +34,7 @@ var allConfigFromEnv = () => {
     [MIDSCENE_OPENAI_INIT_CONFIG_JSON]: process.env[MIDSCENE_OPENAI_INIT_CONFIG_JSON] || void 0,
     [MIDSCENE_MODEL_NAME]: process.env[MIDSCENE_MODEL_NAME] || void 0,
     [MIDSCENE_DEBUG_MODE]: process.env[MIDSCENE_DEBUG_MODE] || void 0,
+    [MIDSCENE_FORCE_DEEP_THINK]: process.env[MIDSCENE_FORCE_DEEP_THINK] || void 0,
     [MIDSCENE_LANGSMITH_DEBUG]: process.env[MIDSCENE_LANGSMITH_DEBUG] || void 0,
     [MIDSCENE_DEBUG_AI_PROFILE]: process.env[MIDSCENE_DEBUG_AI_PROFILE] || void 0,
     [MIDSCENE_DEBUG_AI_RESPONSE]: process.env[MIDSCENE_DEBUG_AI_RESPONSE] || void 0,
@@ -162,6 +164,7 @@ var overrideAIConfig = (newConfig, extendMode) => {
 
 
 
-exports.MIDSCENE_OPENAI_INIT_CONFIG_JSON = MIDSCENE_OPENAI_INIT_CONFIG_JSON; exports.MIDSCENE_MODEL_NAME = MIDSCENE_MODEL_NAME; exports.MIDSCENE_LANGSMITH_DEBUG = MIDSCENE_LANGSMITH_DEBUG; exports.MIDSCENE_DEBUG_AI_PROFILE = MIDSCENE_DEBUG_AI_PROFILE; exports.MIDSCENE_DEBUG_AI_RESPONSE = MIDSCENE_DEBUG_AI_RESPONSE; exports.MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG = MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG; exports.MIDSCENE_DEBUG_MODE = MIDSCENE_DEBUG_MODE; exports.MIDSCENE_OPENAI_SOCKS_PROXY = MIDSCENE_OPENAI_SOCKS_PROXY; exports.OPENAI_API_KEY = OPENAI_API_KEY; exports.OPENAI_BASE_URL = OPENAI_BASE_URL; exports.OPENAI_MAX_TOKENS = OPENAI_MAX_TOKENS; exports.MIDSCENE_CACHE = MIDSCENE_CACHE; exports.MIDSCENE_USE_VLM_UI_TARS = MIDSCENE_USE_VLM_UI_TARS; exports.MIDSCENE_USE_QWEN_VL = MIDSCENE_USE_QWEN_VL; exports.MIDSCENE_USE_DOUBAO_VISION = MIDSCENE_USE_DOUBAO_VISION; exports.MIDSCENE_USE_VL_MODEL = MIDSCENE_USE_VL_MODEL; exports.MATCH_BY_POSITION = MATCH_BY_POSITION; exports.MIDSCENE_API_TYPE = MIDSCENE_API_TYPE; exports.MIDSCENE_REPORT_TAG_NAME = MIDSCENE_REPORT_TAG_NAME; exports.MIDSCENE_USE_AZURE_OPENAI = MIDSCENE_USE_AZURE_OPENAI; exports.MIDSCENE_AZURE_OPENAI_SCOPE = MIDSCENE_AZURE_OPENAI_SCOPE; exports.MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON; exports.AZURE_OPENAI_ENDPOINT = AZURE_OPENAI_ENDPOINT; exports.AZURE_OPENAI_KEY = AZURE_OPENAI_KEY; exports.AZURE_OPENAI_API_VERSION = AZURE_OPENAI_API_VERSION; exports.AZURE_OPENAI_DEPLOYMENT = AZURE_OPENAI_DEPLOYMENT; exports.MIDSCENE_USE_ANTHROPIC_SDK = MIDSCENE_USE_ANTHROPIC_SDK; exports.ANTHROPIC_API_KEY = ANTHROPIC_API_KEY; exports.OPENAI_USE_AZURE = OPENAI_USE_AZURE; exports.vlLocateMode = vlLocateMode; exports.getAIConfig = getAIConfig; exports.getAIConfigInBoolean = getAIConfigInBoolean; exports.getAIConfigInJson = getAIConfigInJson; exports.allAIConfig = allAIConfig; exports.overrideAIConfig = overrideAIConfig;
 
-//# sourceMappingURL=chunk-I2I7OU2E.js.map
+exports.MIDSCENE_OPENAI_INIT_CONFIG_JSON = MIDSCENE_OPENAI_INIT_CONFIG_JSON; exports.MIDSCENE_MODEL_NAME = MIDSCENE_MODEL_NAME; exports.MIDSCENE_LANGSMITH_DEBUG = MIDSCENE_LANGSMITH_DEBUG; exports.MIDSCENE_DEBUG_AI_PROFILE = MIDSCENE_DEBUG_AI_PROFILE; exports.MIDSCENE_DEBUG_AI_RESPONSE = MIDSCENE_DEBUG_AI_RESPONSE; exports.MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG = MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG; exports.MIDSCENE_DEBUG_MODE = MIDSCENE_DEBUG_MODE; exports.MIDSCENE_FORCE_DEEP_THINK = MIDSCENE_FORCE_DEEP_THINK; exports.MIDSCENE_OPENAI_SOCKS_PROXY = MIDSCENE_OPENAI_SOCKS_PROXY; exports.OPENAI_API_KEY = OPENAI_API_KEY; exports.OPENAI_BASE_URL = OPENAI_BASE_URL; exports.OPENAI_MAX_TOKENS = OPENAI_MAX_TOKENS; exports.MIDSCENE_CACHE = MIDSCENE_CACHE; exports.MIDSCENE_USE_VLM_UI_TARS = MIDSCENE_USE_VLM_UI_TARS; exports.MIDSCENE_USE_QWEN_VL = MIDSCENE_USE_QWEN_VL; exports.MIDSCENE_USE_DOUBAO_VISION = MIDSCENE_USE_DOUBAO_VISION; exports.MIDSCENE_USE_VL_MODEL = MIDSCENE_USE_VL_MODEL; exports.MATCH_BY_POSITION = MATCH_BY_POSITION; exports.MIDSCENE_API_TYPE = MIDSCENE_API_TYPE; exports.MIDSCENE_REPORT_TAG_NAME = MIDSCENE_REPORT_TAG_NAME; exports.MIDSCENE_USE_AZURE_OPENAI = MIDSCENE_USE_AZURE_OPENAI; exports.MIDSCENE_AZURE_OPENAI_SCOPE = MIDSCENE_AZURE_OPENAI_SCOPE; exports.MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON; exports.AZURE_OPENAI_ENDPOINT = AZURE_OPENAI_ENDPOINT; exports.AZURE_OPENAI_KEY = AZURE_OPENAI_KEY; exports.AZURE_OPENAI_API_VERSION = AZURE_OPENAI_API_VERSION; exports.AZURE_OPENAI_DEPLOYMENT = AZURE_OPENAI_DEPLOYMENT; exports.MIDSCENE_USE_ANTHROPIC_SDK = MIDSCENE_USE_ANTHROPIC_SDK; exports.ANTHROPIC_API_KEY = ANTHROPIC_API_KEY; exports.OPENAI_USE_AZURE = OPENAI_USE_AZURE; exports.vlLocateMode = vlLocateMode; exports.getAIConfig = getAIConfig; exports.getAIConfigInBoolean = getAIConfigInBoolean; exports.getAIConfigInJson = getAIConfigInJson; exports.allAIConfig = allAIConfig; exports.overrideAIConfig = overrideAIConfig;
+
+//# sourceMappingURL=chunk-UBGEKXK7.js.map

package/dist/lib/chunk-UBGEKXK7.js.map

@@ -0,0 +1 @@
+{"version":3,"mappings":";AACO,IAAM,mCACX;AACK,IAAM,sBAAsB;AAC5B,IAAM,2BAA2B;AACjC,IAAM,4BAA4B;AAClC,IAAM,6BAA6B;AACnC,IAAM,wCACX;AACK,IAAM,sBAAsB;AAE5B,IAAM,4BAA4B;AAElC,IAAM,8BAA8B;AACpC,IAAM,iBAAiB;AACvB,IAAM,kBAAkB;AACxB,IAAM,oBAAoB;AAE1B,IAAM,iBAAiB;AACvB,IAAM,2BAA2B;AACjC,IAAM,uBAAuB;AAC7B,IAAM,6BAA6B;AACnC,IAAM,wBAAwB;AAC9B,IAAM,oBAAoB;AAC1B,IAAM,oBAAoB;AAC1B,IAAM,2BAA2B;AAEjC,IAAM,4BAA4B;AAClC,IAAM,8BAA8B;AACpC,IAAM,yCACX;AAEK,IAAM,wBAAwB;AAC9B,IAAM,mBAAmB;AACzB,IAAM,2BAA2B;AACjC,IAAM,0BAA0B;AAEhC,IAAM,6BAA6B;AACnC,IAAM,oBAAoB;AAG1B,IAAM,mBAAmB;AAEhC,IAAM,mBAAmB,MAAM;AAC7B,SAAO;AAAA,IACL,CAAC,gCAAgC,GAC/B,QAAQ,IAAI,gCAAgC,KAAK;AAAA,IACnD,CAAC,mBAAmB,GAAG,QAAQ,IAAI,mBAAmB,KAAK;AAAA,IAC3D,CAAC,mBAAmB,GAAG,QAAQ,IAAI,mBAAmB,KAAK;AAAA,IAC3D,CAAC,yBAAyB,GACxB,QAAQ,IAAI,yBAAyB,KAAK;AAAA,IAC5C,CAAC,wBAAwB,GACvB,QAAQ,IAAI,wBAAwB,KAAK;AAAA,IAC3C,CAAC,yBAAyB,GACxB,QAAQ,IAAI,yBAAyB,KAAK;AAAA,IAC5C,CAAC,0BAA0B,GACzB,QAAQ,IAAI,0BAA0B,KAAK;AAAA,IAC7C,CAAC,qCAAqC,GACpC,QAAQ,IAAI,qCAAqC,KAAK;AAAA,IACxD,CAAC,cAAc,GAAG,QAAQ,IAAI,cAAc,KAAK;AAAA,IACjD,CAAC,eAAe,GAAG,QAAQ,IAAI,eAAe,KAAK;AAAA,IACnD,CAAC,iBAAiB,GAAG,QAAQ,IAAI,iBAAiB,KAAK;AAAA,IACvD,CAAC,gBAAgB,GAAG,QAAQ,IAAI,gBAAgB,KAAK;AAAA,IACrD,CAAC,cAAc,GAAG,QAAQ,IAAI,cAAc,KAAK;AAAA,IACjD,CAAC,iBAAiB,GAAG,QAAQ,IAAI,iBAAiB,KAAK;AAAA,IACvD,CAAC,wBAAwB,GACvB,QAAQ,IAAI,wBAAwB,KAAK;AAAA,IAC3C,CAAC,2BAA2B,GAC1B,QAAQ,IAAI,2BAA2B,KAAK;AAAA,IAC9C,CAAC,yBAAyB,GACxB,QAAQ,IAAI,yBAAyB,KAAK;AAAA,IAC5C,CAAC,2BAA2B,GAC1B,QAAQ,IAAI,2BAA2B,KAAK;AAAA,IAC9C,CAAC,sCAAsC,GACrC,QAAQ,IAAI,sCAAsC,KAAK;AAAA,IACzD,CAAC,0BAA0B,GACzB,QAAQ,IAAI,0BAA0B,KAAK;AAAA,IAC7C,CAAC,wBAAwB,GACvB,QAAQ,IAAI,wBAAwB,KAAK;AAAA,IAC3C,CAAC,oBAAoB,GAAG,QAAQ,IAAI,oBAAoB,KAAK;AAAA,IAC7D,CAAC,0BAA0B,GACzB,QAAQ,IAAI,0BAA0B,KAAK;AAAA,IAC7C,CAAC,qBAAqB,GAAG,QAAQ,IAAI,qBAAqB,KAAK;AAAA,IAC/D,CAAC,iBAAiB,GAAG,QAAQ,IAAI,iBAAiB,KAAK;AAAA,IACvD,CAAC,qBAAqB,GAAG,QAAQ,IAAI,qBAAqB,KAAK;AAAA,IAC/D,CAAC,gBAAgB,GAAG,QAAQ,IAAI,gBAAgB,KAAK;AAAA,IACrD,CAAC,wBAAwB,GACvB,QAAQ,IAAI,wBAAwB,KAAK;AAAA,IAC3C,CAAC,uBAAuB,GACtB,QAAQ,IAAI,uBAAuB,KAAK;AAAA,EAC5C;AACF;AAEA,IAAI,aAA2D,CAAC;AAEzD,IAAM,eAAe,MAIf;AACX,MACE,qBAAqB,0BAA0B,KAC/C,qBAAqB,oBAAoB,GACzC;AACA,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,MAAI,qBAAqB,oBAAoB,GAAG;AAC9C,WAAO;AAAA,EACT;AAEA,MAAI,qBAAqB,0BAA0B,GAAG;AACpD,WAAO;AAAA,EACT;AAEA,MAAI,qBAAqB,qBAAqB,GAAG;AAC/C,WAAO;AAAA,EACT;AAEA,SAAO;AACT;AAEO,IAAM,cAAc,CACzB,cACuB;AACvB,MAAI,cAAc,mBAAmB;AACnC,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,WAAW,SAAS,MAAM,aAAa;AAChD,QAAI,OAAO,WAAW,SAAS,MAAM,UAAU;AAC7C,aAAO,WAAW,SAAS,GAAG,KAAK;AAAA,IACrC;AACA,WAAO,WAAW,SAAS;AAAA,EAC7B;AACA,SAAO,iBAAiB,EAAE,SAAS,GAAG,KAAK;AAC7C;AAEO,IAAM,uBAAuB,CAAC,cAAuC;AAC1E,QAAM,SAAS,YAAY,SAAS,KAAK;AACzC,SAAO,cAAc,KAAK,MAAM;AAClC;AAEO,IAAM,oBAAoB,CAAC,cAAuC;AACvE,QAAM,SAAS,YAAY,SAAS;AACpC,MAAI;AACF,WAAO,SAAS,KAAK,MAAM,MAAM,IAAI;AAAA,EACvC,SAAS,OAAY;AACnB,UAAM,IAAI;AAAA,MACR,gCAAgC,SAAS,KAAK,MAAM,OAAO;AAAA,MAC3D;AAAA,QACE,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AACF;AAEO,IAAM,cAAc,MAAM;AAC/B,SAAO,EAAE,GAAG,iBAAiB,GAAG,GAAG,WAAW;AAChD;AAEO,IAAM,mBAAmB,CAC9B,WACA,eACG;AACH,aAAW,OAAO,WAAW;AAC3B,QAAI,OAAO,QAAQ,UAAU;AAC3B,YAAM,IAAI,MAAM,8CAA8C,GAAG,EAAE;AAAA,IACrE;AACA,QAAI,OAAO,UAAU,GAA6B,MAAM,UAAU;AAChE,YAAM,IAAI;AAAA,QACR,wDAAwD,GAAG,YAAY,UAAU,GAA6B,CAAC;AAAA,MACjH;AAAA,IACF;AAAA,EACF;AAEA,eAAa,aAAa,EAAE,GAAG,YAAY,GAAG,UAAU,IAAI,EAAE,GAAG,UAAU;AAC7E","names":[],"ignoreList":[],"sources":["../../src/env.ts"],"sourcesContent":["// config keys\nexport const MIDSCENE_OPENAI_INIT_CONFIG_JSON =\n  'MIDSCENE_OPENAI_INIT_CONFIG_JSON';\nexport const MIDSCENE_MODEL_NAME = 'MIDSCENE_MODEL_NAME';\nexport const MIDSCENE_LANGSMITH_DEBUG = 'MIDSCENE_LANGSMITH_DEBUG';\nexport const MIDSCENE_DEBUG_AI_PROFILE = 'MIDSCENE_DEBUG_AI_PROFILE';\nexport const MIDSCENE_DEBUG_AI_RESPONSE = 'MIDSCENE_DEBUG_AI_RESPONSE';\nexport const MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG =\n  'MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG';\nexport const MIDSCENE_DEBUG_MODE = 'MIDSCENE_DEBUG_MODE';\n\nexport const MIDSCENE_FORCE_DEEP_THINK = 'MIDSCENE_FORCE_DEEP_THINK';\n\nexport const MIDSCENE_OPENAI_SOCKS_PROXY = 'MIDSCENE_OPENAI_SOCKS_PROXY';\nexport const OPENAI_API_KEY = 'OPENAI_API_KEY';\nexport const OPENAI_BASE_URL = 'OPENAI_BASE_URL';\nexport const OPENAI_MAX_TOKENS = 'OPENAI_MAX_TOKENS';\n\nexport const MIDSCENE_CACHE = 'MIDSCENE_CACHE';\nexport const MIDSCENE_USE_VLM_UI_TARS = 'MIDSCENE_USE_VLM_UI_TARS';\nexport const MIDSCENE_USE_QWEN_VL = 'MIDSCENE_USE_QWEN_VL';\nexport const MIDSCENE_USE_DOUBAO_VISION = 'MIDSCENE_USE_DOUBAO_VISION';\nexport const MIDSCENE_USE_VL_MODEL = 'MIDSCENE_USE_VL_MODEL';\nexport const MATCH_BY_POSITION = 'MATCH_BY_POSITION';\nexport const MIDSCENE_API_TYPE = 'MIDSCENE-API-TYPE';\nexport const MIDSCENE_REPORT_TAG_NAME = 'MIDSCENE_REPORT_TAG_NAME';\n\nexport const MIDSCENE_USE_AZURE_OPENAI = 'MIDSCENE_USE_AZURE_OPENAI';\nexport const MIDSCENE_AZURE_OPENAI_SCOPE = 'MIDSCENE_AZURE_OPENAI_SCOPE';\nexport const MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON =\n  'MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON';\n\nexport const AZURE_OPENAI_ENDPOINT = 'AZURE_OPENAI_ENDPOINT';\nexport const AZURE_OPENAI_KEY = 'AZURE_OPENAI_KEY';\nexport const AZURE_OPENAI_API_VERSION = 'AZURE_OPENAI_API_VERSION';\nexport const AZURE_OPENAI_DEPLOYMENT = 'AZURE_OPENAI_DEPLOYMENT';\n\nexport const MIDSCENE_USE_ANTHROPIC_SDK = 'MIDSCENE_USE_ANTHROPIC_SDK';\nexport const ANTHROPIC_API_KEY = 'ANTHROPIC_API_KEY';\n\n// @deprecated\nexport const OPENAI_USE_AZURE = 'OPENAI_USE_AZURE';\n\nconst allConfigFromEnv = () => {\n  return {\n    [MIDSCENE_OPENAI_INIT_CONFIG_JSON]:\n      process.env[MIDSCENE_OPENAI_INIT_CONFIG_JSON] || undefined,\n    [MIDSCENE_MODEL_NAME]: process.env[MIDSCENE_MODEL_NAME] || undefined,\n    [MIDSCENE_DEBUG_MODE]: process.env[MIDSCENE_DEBUG_MODE] || undefined,\n    [MIDSCENE_FORCE_DEEP_THINK]:\n      process.env[MIDSCENE_FORCE_DEEP_THINK] || undefined,\n    [MIDSCENE_LANGSMITH_DEBUG]:\n      process.env[MIDSCENE_LANGSMITH_DEBUG] || undefined,\n    [MIDSCENE_DEBUG_AI_PROFILE]:\n      process.env[MIDSCENE_DEBUG_AI_PROFILE] || undefined,\n    [MIDSCENE_DEBUG_AI_RESPONSE]:\n      process.env[MIDSCENE_DEBUG_AI_RESPONSE] || undefined,\n    [MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG]:\n      process.env[MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG] || undefined,\n    [OPENAI_API_KEY]: process.env[OPENAI_API_KEY] || undefined,\n    [OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || undefined,\n    [OPENAI_MAX_TOKENS]: process.env[OPENAI_MAX_TOKENS] || undefined,\n    [OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || undefined,\n    [MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || undefined,\n    [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || undefined,\n    [MIDSCENE_REPORT_TAG_NAME]:\n      process.env[MIDSCENE_REPORT_TAG_NAME] || undefined,\n    [MIDSCENE_OPENAI_SOCKS_PROXY]:\n      process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || undefined,\n    [MIDSCENE_USE_AZURE_OPENAI]:\n      process.env[MIDSCENE_USE_AZURE_OPENAI] || undefined,\n    [MIDSCENE_AZURE_OPENAI_SCOPE]:\n      process.env[MIDSCENE_AZURE_OPENAI_SCOPE] || undefined,\n    [MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON]:\n      process.env[MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON] || undefined,\n    [MIDSCENE_USE_ANTHROPIC_SDK]:\n      process.env[MIDSCENE_USE_ANTHROPIC_SDK] || undefined,\n    [MIDSCENE_USE_VLM_UI_TARS]:\n      process.env[MIDSCENE_USE_VLM_UI_TARS] || undefined,\n    [MIDSCENE_USE_QWEN_VL]: process.env[MIDSCENE_USE_QWEN_VL] || undefined,\n    [MIDSCENE_USE_DOUBAO_VISION]:\n      process.env[MIDSCENE_USE_DOUBAO_VISION] || undefined,\n    [MIDSCENE_USE_VL_MODEL]: process.env[MIDSCENE_USE_VL_MODEL] || undefined,\n    [ANTHROPIC_API_KEY]: process.env[ANTHROPIC_API_KEY] || undefined,\n    [AZURE_OPENAI_ENDPOINT]: process.env[AZURE_OPENAI_ENDPOINT] || undefined,\n    [AZURE_OPENAI_KEY]: process.env[AZURE_OPENAI_KEY] || undefined,\n    [AZURE_OPENAI_API_VERSION]:\n      process.env[AZURE_OPENAI_API_VERSION] || undefined,\n    [AZURE_OPENAI_DEPLOYMENT]:\n      process.env[AZURE_OPENAI_DEPLOYMENT] || undefined,\n  };\n};\n\nlet userConfig: Partial<ReturnType<typeof allConfigFromEnv>> = {};\n\nexport const vlLocateMode = ():\n  | 'qwen-vl'\n  | 'doubao-vision'\n  | 'vl-model'\n  | false => {\n  if (\n    getAIConfigInBoolean(MIDSCENE_USE_DOUBAO_VISION) &&\n    getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)\n  ) {\n    throw new Error(\n      'MIDSCENE_USE_DOUBAO_VISION and MIDSCENE_USE_QWEN_VL cannot be true at the same time',\n    );\n  }\n\n  if (getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)) {\n    return 'qwen-vl';\n  }\n\n  if (getAIConfigInBoolean(MIDSCENE_USE_DOUBAO_VISION)) {\n    return 'doubao-vision';\n  }\n\n  if (getAIConfigInBoolean(MIDSCENE_USE_VL_MODEL)) {\n    return 'vl-model';\n  }\n\n  return false;\n};\n\nexport const getAIConfig = (\n  configKey: keyof typeof userConfig,\n): string | undefined => {\n  if (configKey === MATCH_BY_POSITION) {\n    throw new Error(\n      'MATCH_BY_POSITION is deprecated, use MIDSCENE_USE_VL_MODEL instead',\n    );\n  }\n\n  if (typeof userConfig[configKey] !== 'undefined') {\n    if (typeof userConfig[configKey] === 'string') {\n      return userConfig[configKey]?.trim();\n    }\n    return userConfig[configKey];\n  }\n  return allConfigFromEnv()[configKey]?.trim();\n};\n\nexport const getAIConfigInBoolean = (configKey: keyof typeof userConfig) => {\n  const config = getAIConfig(configKey) || '';\n  return /^(true|1)$/i.test(config);\n};\n\nexport const getAIConfigInJson = (configKey: keyof typeof userConfig) => {\n  const config = getAIConfig(configKey);\n  try {\n    return config ? JSON.parse(config) : undefined;\n  } catch (error: any) {\n    throw new Error(\n      `Failed to parse json config: ${configKey}. ${error.message}`,\n      {\n        cause: error,\n      },\n    );\n  }\n};\n\nexport const allAIConfig = () => {\n  return { ...allConfigFromEnv(), ...userConfig };\n};\n\nexport const overrideAIConfig = (\n  newConfig: Partial<ReturnType<typeof allConfigFromEnv>>,\n  extendMode?: boolean,\n) => {\n  for (const key in newConfig) {\n    if (typeof key !== 'string') {\n      throw new Error(`Failed to override AI config, invalid key: ${key}`);\n    }\n    if (typeof newConfig[key as keyof typeof newConfig] === 'object') {\n      throw new Error(\n        `Failed to override AI config, invalid value for key: ${key}, value: ${newConfig[key as keyof typeof newConfig]}`,\n      );\n    }\n  }\n\n  userConfig = extendMode ? { ...userConfig, ...newConfig } : { ...newConfig };\n};\n"]}