@midscene/core 0.11.4-beta-20250220160619.0 → 0.12.0

package/dist/lib/ai-model.js

@@ -9,7 +9,7 @@
 
 
 
-var _chunk7ROPU2SYjs = require('./chunk-7ROPU2SY.js');
+var _chunkYOJJI4TIjs = require('./chunk-YOJJI4TI.js');
 require('./chunk-WVG2QNX6.js');
 require('./chunk-LQ2VYIYD.js');
 
@@ -23,4 +23,4 @@ require('./chunk-LQ2VYIYD.js');
 
 
 
-exports.AiAssert = _chunk7ROPU2SYjs.AiAssert; exports.AiExtractElementInfo = _chunk7ROPU2SYjs.AiExtractElementInfo; exports.AiInspectElement = _chunk7ROPU2SYjs.AiInspectElement; exports.callAiFn = _chunk7ROPU2SYjs.callAiFn; exports.callToGetJSONObject = _chunk7ROPU2SYjs.callToGetJSONObject; exports.describeUserPage = _chunk7ROPU2SYjs.describeUserPage; exports.plan = _chunk7ROPU2SYjs.plan; exports.systemPromptToLocateElement = _chunk7ROPU2SYjs.systemPromptToLocateElement; exports.transformElementPositionToId = _chunk7ROPU2SYjs.transformElementPositionToId; exports.vlmPlanning = _chunk7ROPU2SYjs.vlmPlanning;
+exports.AiAssert = _chunkYOJJI4TIjs.AiAssert; exports.AiExtractElementInfo = _chunkYOJJI4TIjs.AiExtractElementInfo; exports.AiInspectElement = _chunkYOJJI4TIjs.AiInspectElement; exports.callAiFn = _chunkYOJJI4TIjs.callAiFn; exports.callToGetJSONObject = _chunkYOJJI4TIjs.callToGetJSONObject; exports.describeUserPage = _chunkYOJJI4TIjs.describeUserPage; exports.plan = _chunkYOJJI4TIjs.plan; exports.systemPromptToLocateElement = _chunkYOJJI4TIjs.systemPromptToLocateElement; exports.transformElementPositionToId = _chunkYOJJI4TIjs.transformElementPositionToId; exports.vlmPlanning = _chunkYOJJI4TIjs.vlmPlanning;

package/dist/lib/{chunk-Z35NCVLT.js → chunk-S4BDFBOI.js}

@@ -2,13 +2,13 @@
 
 
 
-var _chunkP3XQH42Cjs = require('./chunk-P3XQH42C.js');
+var _chunkXAXHXB3Sjs = require('./chunk-XAXHXB3S.js');
 
 
 
 
 
-var _chunk7ROPU2SYjs = require('./chunk-7ROPU2SY.js');
+var _chunkYOJJI4TIjs = require('./chunk-YOJJI4TI.js');
 
 
 
@@ -162,7 +162,7 @@ ${(_b = this.latestErrorTask()) == null ? void 0 : _b.errorStack}`
   }
   dump() {
     const dumpData = {
-      sdkVersion: _chunkP3XQH42Cjs.getVersion.call(void 0, ),
+      sdkVersion: _chunkXAXHXB3Sjs.getVersion.call(void 0, ),
       model_name: _chunkWVG2QNX6js.getAIConfig.call(void 0, _chunkWVG2QNX6js.MIDSCENE_MODEL_NAME) || "",
       logTime: Date.now(),
       name: this.name,
@@ -181,7 +181,7 @@ var _utils = require('@midscene/shared/utils');
 var logContent = [];
 var logIdIndexMap = {};
 function emitInsightDump(data, logId, dumpSubscriber) {
-  const logDir = _chunkP3XQH42Cjs.getLogDir.call(void 0, );
+  const logDir = _chunkXAXHXB3Sjs.getLogDir.call(void 0, );
   _assert2.default.call(void 0, logDir, "logDir should be set before writing dump file");
   const id = logId || _utils.uuid.call(void 0, );
   let modelDescription = "";
@@ -191,7 +191,7 @@ function emitInsightDump(data, logId, dumpSubscriber) {
     modelDescription = "qwen-vl mode";
   }
   const baseData = {
-    sdkVersion: _chunkP3XQH42Cjs.getVersion.call(void 0, ),
+    sdkVersion: _chunkXAXHXB3Sjs.getVersion.call(void 0, ),
     logTime: Date.now(),
     model_name: _chunkWVG2QNX6js.getAIConfig.call(void 0, _chunkWVG2QNX6js.MIDSCENE_MODEL_NAME) || "",
     model_description: modelDescription
@@ -202,7 +202,7 @@ function emitInsightDump(data, logId, dumpSubscriber) {
     ...data
   };
   dumpSubscriber == null ? void 0 : dumpSubscriber(finalData);
-  const dataString = _chunkP3XQH42Cjs.stringifyDumpData.call(void 0, finalData, 2);
+  const dataString = _chunkXAXHXB3Sjs.stringifyDumpData.call(void 0, finalData, 2);
   if (typeof logIdIndexMap[id] === "number") {
     logContent[logIdIndexMap[id]] = dataString;
   } else {
@@ -215,7 +215,7 @@ function emitInsightDump(data, logId, dumpSubscriber) {
 // src/insight/index.ts
 var Insight = class {
   constructor(context, opt) {
-    this.aiVendorFn = _chunk7ROPU2SYjs.callAiFn;
+    this.aiVendorFn = _chunkYOJJI4TIjs.callAiFn;
     _assert2.default.call(void 0, context, "context is required for Insight");
     if (typeof context === "function") {
       this.contextRetrieverFn = context;
@@ -241,7 +241,7 @@ var Insight = class {
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn("locate");
     const startTime = Date.now();
-    const { parseResult, elementById, rawResponse, usage } = await _chunk7ROPU2SYjs.AiInspectElement.call(void 0, {
+    const { parseResult, elementById, rawResponse, usage } = await _chunkYOJJI4TIjs.AiInspectElement.call(void 0, {
       callAI: callAI || this.aiVendorFn,
       context,
       targetElementDescription: queryPrompt,
@@ -320,7 +320,7 @@ ${parseResult.errors.join("\n")}`;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn("extract");
     const startTime = Date.now();
-    const { parseResult, usage } = await _chunk7ROPU2SYjs.AiExtractElementInfo.call(void 0, {
+    const { parseResult, usage } = await _chunkYOJJI4TIjs.AiExtractElementInfo.call(void 0, {
       context,
       dataQuery: dataDemand
     });
@@ -377,7 +377,7 @@ ${parseResult.errors.join("\n")}`;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn("assert");
     const startTime = Date.now();
-    const assertResult = await _chunk7ROPU2SYjs.AiAssert.call(void 0, {
+    const assertResult = await _chunkYOJJI4TIjs.AiAssert.call(void 0, {
       assertion,
       context
     });
@@ -420,4 +420,4 @@ var src_default = Insight;
 
 exports.Executor = Executor; exports.Insight = Insight; exports.src_default = src_default;
 
-//# sourceMappingURL=chunk-Z35NCVLT.js.map
+//# sourceMappingURL=chunk-S4BDFBOI.js.map

package/dist/lib/{chunk-Z35NCVLT.js.map → chunk-S4BDFBOI.js.map}

@@ -1 +1 @@
-{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,OAAO,YAAY;AAaZ,IAAM,WAAN,MAAe;AAAA,EAUpB,YACE,MACA,SAGA;AACA,SAAK,UACH,mCAAS,UAAS,QAAQ,MAAM,SAAS,IAAI,YAAY;AAC3D,SAAK,OAAO;AACZ,SAAK,UAAS,mCAAS,UAAS,CAAC,GAAG;AAAA,MAAI,CAAC,SACvC,KAAK,kBAAkB,IAAI;AAAA,IAC7B;AACA,SAAK,cAAc,mCAAS;AAAA,EAC9B;AAAA,EAEQ,kBAAkB,MAAyC;AACjE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAgE;AA7C/E;AA8CI;AAAA,MACE,KAAK,WAAW;AAAA,MAChB;AAAA,SAAyD,UAAK,gBAAgB,MAArB,mBAAwB,KAAK;AAAA,GAAK,UAAK,gBAAgB,MAArB,mBAAwB,UAAU;AAAA,IAC/H;AACA,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAK,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,SAAS,KAAK,kBAAkB,IAAI,CAAC,CAAC;AAAA,IACrE,OAAO;AACL,WAAK,MAAM,KAAK,KAAK,kBAAkB,IAAI,CAAC;AAAA,IAC9C;AACA,QAAI,KAAK,WAAW,WAAW;AAC7B,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,QAAsB;AAC1B,QAAI,KAAK,WAAW,UAAU,KAAK,MAAM,SAAS,GAAG;AACnD,cAAQ;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,WAAW,WAAW,6BAA6B;AAC/D,WAAO,KAAK,WAAW,aAAa,+BAA+B;AACnE,WAAO,KAAK,WAAW,SAAS,4BAA4B;AAE5D,UAAM,mBAAmB,KAAK,MAAM;AAAA,MAClC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,mBAAmB,GAAG;AAExB;AAAA,IACF;AAEA,SAAK,SAAS;AACd,QAAI,YAAY;AAChB,QAAI,wBAAwB;AAE5B,QAAI;AAEJ,WAAO,YAAY,KAAK,MAAM,QAAQ;AACpC,YAAM,OAAO,KAAK,MAAM,SAAS;AACjC;AAAA,QACE,KAAK,WAAW;AAAA,QAChB,2CAA2C,KAAK,MAAM;AAAA,MACxD;AACA,WAAK,SAAS;AAAA,QACZ,OAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI;AACF,aAAK,SAAS;AACd,YAAI;AACF,cAAI,KAAK,aAAa;AACpB,kBAAM,KAAK,YAAY,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ,MAAM,wBAAwB,CAAC;AAAA,QACzC;AACA;AAAA,UACE,CAAC,WAAW,UAAU,UAAU,EAAE,QAAQ,KAAK,IAAI,KAAK;AAAA,UACxD,0BAA0B,KAAK,IAAI;AAAA,QACrC;AAEA,cAAM,EAAE,UAAU,MAAM,IAAI;AAC5B,eAAO,UAAU,uCAAuC,KAAK,IAAI,EAAE;AAEnE,YAAI;AACJ,cAAM,kBAAmC;AAAA,UACvC;AAAA,UACA,SAAS,yDAAoB;AAAA,QAC/B;AAEA,YAAI,KAAK,SAAS,WAAW;AAC3B;AAAA,YACE,KAAK,YAAY,YACf,KAAK,YAAY,WACjB,KAAK,YAAY;AAAA,YACnB,gCAAgC,KAAK,OAAO;AAAA,UAC9C;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AACxD,cAAI,KAAK,YAAY,UAAU;AAC7B,iCACE,2CACC;AAAA,UACL;AAAA,QACF,WAAW,KAAK,SAAS,YAAY,KAAK,SAAS,YAAY;AAC7D,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D,OAAO;AACL,kBAAQ;AAAA,YACN,0BAA0B,KAAK,IAAI;AAAA,UACrC;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D;AAEA,eAAO,OAAO,MAAM,WAAW;AAC/B,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD,aAAK,OAAO,UAAU,2CAAqB,WAAU;AACrD;AAAA,MACF,SAAS,GAAQ;AACf,gCAAwB;AACxB,aAAK,SACH,uBAAG,aAAY,OAAO,MAAM,WAAW,IAAI;AAC7C,aAAK,aAAa,EAAE;AAEpB,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF;AAAA,IACF;AAGA,aAAS,IAAI,YAAY,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AACtD,WAAK,MAAM,CAAC,EAAE,SAAS;AAAA,IACzB;AAEA,QAAI,uBAAuB;AACzB,WAAK,SAAS;AAAA,IAChB,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAEA,QAAI,KAAK,MAAM,QAAQ;AAErB,YAAM,cAAc,KAAK,IAAI,WAAW,KAAK,MAAM,SAAS,CAAC;AAC7D,aAAO,KAAK,MAAM,WAAW,EAAE;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,iBAA0B;AACxB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,kBAAwC;AACtC,QAAI,KAAK,WAAW,SAAS;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,iBAAiB,KAAK,MAAM;AAAA,MAChC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,kBAAkB,GAAG;AACvB,aAAO,KAAK,MAAM,cAAc;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAsB;AACpB,UAAM,WAA0B;AAAA,MAC9B,YAAY,WAAW;AAAA,MACvB,YAAY,YAAY,mBAAmB,KAAK;AAAA,MAChD,SAAS,KAAK,IAAI;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,IACd;AACA,WAAO;AAAA,EACT;AACF;;;AC3MA,OAAOA,aAAY;;;ACAnB,OAAOA,aAAY;AAenB,SAAS,YAAY;AACrB,IAAM,aAAuB,CAAC;AAC9B,IAAM,gBAAwC,CAAC;AAExC,SAAS,gBACd,MACA,OACA,gBACQ;AACR,QAAM,SAAS,UAAU;AACzB,EAAAA,QAAO,QAAQ,+CAA+C;AAE9D,QAAM,KAAK,SAAS,KAAK;AACzB,MAAI,mBAAmB;AACvB,MAAI,qBAAqB,wBAAwB,GAAG;AAClD,uBAAmB;AAAA,EACrB,WAAW,qBAAqB,oBAAoB,GAAG;AACrD,uBAAmB;AAAA,EACrB;AACA,QAAM,WAAqB;AAAA,IACzB,YAAY,WAAW;AAAA,IACvB,SAAS,KAAK,IAAI;AAAA,IAClB,YAAY,YAAY,mBAAmB,KAAK;AAAA,IAChD,mBAAmB;AAAA,EACrB;AACA,QAAM,YAAyB;AAAA,IAC7B,OAAO;AAAA,IACP,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAEA,mDAAiB;AAEjB,QAAM,aAAa,kBAAkB,WAAW,CAAC;AAEjD,MAAI,OAAO,cAAc,EAAE,MAAM,UAAU;AACzC,eAAW,cAAc,EAAE,CAAC,IAAI;AAAA,EAClC,OAAO;AACL,UAAM,SAAS,WAAW,KAAK,UAAU;AACzC,kBAAc,EAAE,IAAI,SAAS;AAAA,EAC/B;AAEA,SAAO;AACT;;;AD9BA,IAAqB,UAArB,MAGE;AAAA,EAaA,YACE,SAGA,KACA;AAbF,sBAAoD;AAclD,IAAAA,QAAO,SAAS,iCAAiC;AACjD,QAAI,OAAO,YAAY,YAAY;AACjC,WAAK,qBAAqB;AAAA,IAC5B,OAAO;AACL,WAAK,qBAAqB,MAAM,QAAQ,QAAQ,OAAO;AAAA,IACzD;AAEA,SAAK,kBAAkB,2BAAK;AAE5B,QAAI,QAAO,2BAAK,gBAAe,aAAa;AAC1C,WAAK,aAAa,IAAI;AAAA,IACxB;AACA,QAAI,QAAO,2BAAK,cAAa,aAAa;AACxC,WAAK,WAAW,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAMA,MAAM,OAAO,aAAqB,KAAkB;AAvEtD;AAwEI,UAAM,EAAE,OAAO,IAAI,OAAO,CAAC;AAC3B,IAAAA;AAAA,MACE,gBAAe,2BAAK;AAAA,MACpB;AAAA,IACF;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AACzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AAEtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,aAAa,aAAa,MAAM,IACnD,MAAM,iBAAiB;AAAA,MACrB,QAAQ,UAAU,KAAK;AAAA,MACvB;AAAA,MACA,0BAA0B;AAAA,MAC1B,aAAa,2BAAK;AAAA,IACpB,CAAC;AAEH,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,MACvC,gBAAgB,KAAK,UAAU,WAAW;AAAA,MAC1C;AAAA,IACF;AAEA,QAAI;AACJ,SAAI,iBAAY,WAAZ,mBAAoB,QAAQ;AAC9B,iBAAW;AAAA,EAAiC,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAC3E;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT,SAAS;AAAA,MACX;AAAA,MACA,aAAa,2BAAK;AAAA,MAClB,gBAAgB,CAAC;AAAA,MACjB,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,QAAQ,gBAAgB,UAAU,QAAW,cAAc;AAEjE,QAAI,UAAU;AACZ,cAAQ,MAAM,QAAQ;AACtB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,UAAM,WAA0B,CAAC;AACjC,gBAAY,SAAS,QAAQ,CAAC,SAAS;AACrC,UAAI,QAAQ,MAAM;AAChB,cAAM,UAAU,YAAY,KAAK,EAAE;AAEnC,YAAI,CAAC,SAAS;AACZ,kBAAQ;AAAA,YACN,kCAAkC,KAAK,EAAE;AAAA,UAC3C;AACA;AAAA,QACF;AACA,iBAAS,KAAK,OAAO;AAAA,MACvB;AAAA,IACF,CAAC;AAED;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI,SAAS,UAAU,GAAG;AACxB,cAAQ;AAAA,QACN,kEAAkE,WAAW;AAAA,MAC/E;AACA,aAAO,SAAS,CAAC;AAAA,IACnB;AACA,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO,SAAS,CAAC;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAQA,MAAM,QAAW,YAA+C;AAvKlE;AAwKI,IAAAA;AAAA,MACE,OAAO,eAAe,YAAY,OAAO,eAAe;AAAA,MACxD,kDAAkD,OAAO,UAAU;AAAA,IACrE;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,SAAS;AAEvD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,IAAI,MAAM,qBAAwB;AAAA,MAC3D;AAAA,MACA,WAAW;AAAA,IACb,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,IACzC;AAEA,QAAI;AACJ,SAAI,iBAAY,WAAZ,mBAAoB,QAAQ;AAC9B,iBAAW;AAAA,EAAwB,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAClE;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AACA,UAAM,QAAQ,gBAAgB,UAAU,QAAW,cAAc;AAEjE,UAAM,EAAE,KAAK,IAAI;AACjB,QAAI,YAAY,CAAC,MAAM;AACrB,cAAQ,MAAM,QAAQ;AACtB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB,CAAC;AAAA,QACjB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,WAAsD;AACjE,QAAI,OAAO,cAAc,UAAU;AACjC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AACtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,aAAa,OAAO;AAAA,IAClD;AAEA,UAAM,EAAE,SAAS,KAAK,IAAI,aAAa;AACvC,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,eAAe;AAAA,MACf,kBAAkB;AAAA,MAClB,OAAO,OAAO,SAAY;AAAA,IAC5B;AACA,oBAAgB,UAAU,QAAW,cAAc;AAEnD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AACF;;;AEvQA,IAAO,cAAQ","names":["assert"],"ignoreList":[],"sources":["../../src/ai-model/action-executor.ts","../../src/insight/index.ts","../../src/insight/utils.ts","../../src/index.ts"],"sourcesContent":["import assert from 'node:assert';\nimport { MIDSCENE_MODEL_NAME, getAIConfig } from '@/env';\nimport type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<any> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        task.timing.aiCost = (returnValue as any)?.aiCost || 0;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      return this.tasks[outputIndex].output;\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n","import assert from 'node:assert';\nimport { callAiFn } from '@/ai-model/common';\nimport { AiExtractElementInfo, AiInspectElement } from '@/ai-model/index';\nimport { AiAssert } from '@/ai-model/inspect';\nimport type {\n  AIElementResponse,\n  AISingleElementResponse,\n  BaseElement,\n  DumpSubscriber,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  PartialInsightDumpFromSDK,\n  UIContext,\n} from '@/types';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  callAI?: typeof callAiFn<AIElementResponse>;\n  quickAnswer?: Partial<AISingleElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  generateElement: InsightOptions['generateElement'];\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    this.generateElement = opt?.generateElement;\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    queryPrompt: string,\n    opt?: LocateOpts,\n  ): Promise<ElementType | null>;\n  async locate(queryPrompt: string, opt?: LocateOpts) {\n    const { callAI } = opt || {};\n    assert(\n      queryPrompt || opt?.quickAnswer,\n      'query or quickAnswer is required for locate',\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n    const context = await this.contextRetrieverFn('locate');\n\n    const startTime = Date.now();\n    const { parseResult, elementById, rawResponse, usage } =\n      await AiInspectElement({\n        callAI: callAI || this.aiVendorFn,\n        context,\n        targetElementDescription: queryPrompt,\n        quickAnswer: opt?.quickAnswer,\n      });\n    // const parseResult = await this.aiVendorFn<AIElementParseResponse>(msgs);\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `locate - AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      context,\n      userQuery: {\n        element: queryPrompt,\n      },\n      quickAnswer: opt?.quickAnswer,\n      matchedSection: [],\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const logId = emitInsightDump(dumpData, undefined, dumpSubscriber);\n\n    if (errorLog) {\n      console.error(errorLog);\n      throw new Error(errorLog);\n    }\n\n    const elements: BaseElement[] = [];\n    parseResult.elements.forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      logId,\n      dumpSubscriber,\n    );\n\n    if (elements.length >= 2) {\n      console.warn(\n        `locate: multiple elements found, return the first one. (query: ${queryPrompt})`,\n      );\n      return elements[0];\n    }\n    if (elements.length === 1) {\n      return elements[0];\n    }\n    return null;\n  }\n\n  async extract<T = any>(input: string): Promise<T>;\n  async extract<T extends Record<string, string>>(\n    input: T,\n  ): Promise<Record<keyof T, any>>;\n  async extract<T extends object>(input: Record<keyof T, string>): Promise<T>;\n\n  async extract<T>(dataDemand: InsightExtractParam): Promise<any> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      context,\n      userQuery: {\n        dataDemand,\n      },\n      matchedSection: [],\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n    const logId = emitInsightDump(dumpData, undefined, dumpSubscriber);\n\n    const { data } = parseResult;\n    if (errorLog && !data) {\n      console.error(errorLog);\n      throw new Error(errorLog);\n    }\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedSection: [],\n        data,\n      },\n      logId,\n      dumpSubscriber,\n    );\n\n    return {\n      data,\n      usage,\n    };\n  }\n\n  async assert(assertion: string): Promise<InsightAssertionResponse> {\n    if (typeof assertion !== 'string') {\n      throw new Error(\n        'This is the assert method for Midscene, the first argument should be a string. If you want to use the assert method from Node.js, please import it from the Node.js assert module.',\n      );\n    }\n\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      context,\n      userQuery: {\n        assertion,\n      },\n      matchedSection: [],\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, undefined, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n}\n","import assert from 'node:assert';\nimport {\n  MIDSCENE_MODEL_NAME,\n  MIDSCENE_USE_QWEN_VL,\n  MIDSCENE_USE_VLM_UI_TARS,\n  getAIConfig,\n  getAIConfigInBoolean,\n} from '@/env';\nimport type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getLogDir, getVersion, stringifyDumpData } from '@/utils';\nimport { uuid } from '@midscene/shared/utils';\nconst logContent: string[] = [];\nconst logIdIndexMap: Record<string, number> = {};\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  logId?: string,\n  dumpSubscriber?: DumpSubscriber,\n): string {\n  const logDir = getLogDir();\n  assert(logDir, 'logDir should be set before writing dump file');\n\n  const id = logId || uuid();\n  let modelDescription = '';\n  if (getAIConfigInBoolean(MIDSCENE_USE_VLM_UI_TARS)) {\n    modelDescription = 'vlm-ui-tars mode';\n  } else if (getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)) {\n    modelDescription = 'qwen-vl mode';\n  }\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n    model_description: modelDescription,\n  };\n  const finalData: InsightDump = {\n    logId: id,\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n\n  const dataString = stringifyDumpData(finalData, 2);\n\n  if (typeof logIdIndexMap[id] === 'number') {\n    logContent[logIdIndexMap[id]] = dataString;\n  } else {\n    const length = logContent.push(dataString);\n    logIdIndexMap[id] = length - 1;\n  }\n\n  return id;\n}\n","import { Executor } from './ai-model/action-executor';\nimport Insight from './insight';\nimport { getLogDirByType, getVersion, setLogDir } from './utils';\n\nexport {\n  plan,\n  transformElementPositionToId,\n  describeUserPage,\n  AiInspectElement,\n  AiAssert,\n} from './ai-model';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from './env';\n\nexport * from './types';\nexport default Insight;\nexport { Executor, setLogDir, getLogDirByType, Insight, getVersion };\n"]}
+{"version":3,"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,OAAO,YAAY;AAaZ,IAAM,WAAN,MAAe;AAAA,EAUpB,YACE,MACA,SAGA;AACA,SAAK,UACH,mCAAS,UAAS,QAAQ,MAAM,SAAS,IAAI,YAAY;AAC3D,SAAK,OAAO;AACZ,SAAK,UAAS,mCAAS,UAAS,CAAC,GAAG;AAAA,MAAI,CAAC,SACvC,KAAK,kBAAkB,IAAI;AAAA,IAC7B;AACA,SAAK,cAAc,mCAAS;AAAA,EAC9B;AAAA,EAEQ,kBAAkB,MAAyC;AACjE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAgE;AA7C/E;AA8CI;AAAA,MACE,KAAK,WAAW;AAAA,MAChB;AAAA,SAAyD,UAAK,gBAAgB,MAArB,mBAAwB,KAAK;AAAA,GAAK,UAAK,gBAAgB,MAArB,mBAAwB,UAAU;AAAA,IAC/H;AACA,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAK,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,SAAS,KAAK,kBAAkB,IAAI,CAAC,CAAC;AAAA,IACrE,OAAO;AACL,WAAK,MAAM,KAAK,KAAK,kBAAkB,IAAI,CAAC;AAAA,IAC9C;AACA,QAAI,KAAK,WAAW,WAAW;AAC7B,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,QAAsB;AAC1B,QAAI,KAAK,WAAW,UAAU,KAAK,MAAM,SAAS,GAAG;AACnD,cAAQ;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,WAAW,WAAW,6BAA6B;AAC/D,WAAO,KAAK,WAAW,aAAa,+BAA+B;AACnE,WAAO,KAAK,WAAW,SAAS,4BAA4B;AAE5D,UAAM,mBAAmB,KAAK,MAAM;AAAA,MAClC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,mBAAmB,GAAG;AAExB;AAAA,IACF;AAEA,SAAK,SAAS;AACd,QAAI,YAAY;AAChB,QAAI,wBAAwB;AAE5B,QAAI;AAEJ,WAAO,YAAY,KAAK,MAAM,QAAQ;AACpC,YAAM,OAAO,KAAK,MAAM,SAAS;AACjC;AAAA,QACE,KAAK,WAAW;AAAA,QAChB,2CAA2C,KAAK,MAAM;AAAA,MACxD;AACA,WAAK,SAAS;AAAA,QACZ,OAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI;AACF,aAAK,SAAS;AACd,YAAI;AACF,cAAI,KAAK,aAAa;AACpB,kBAAM,KAAK,YAAY,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ,MAAM,wBAAwB,CAAC;AAAA,QACzC;AACA;AAAA,UACE,CAAC,WAAW,UAAU,UAAU,EAAE,QAAQ,KAAK,IAAI,KAAK;AAAA,UACxD,0BAA0B,KAAK,IAAI;AAAA,QACrC;AAEA,cAAM,EAAE,UAAU,MAAM,IAAI;AAC5B,eAAO,UAAU,uCAAuC,KAAK,IAAI,EAAE;AAEnE,YAAI;AACJ,cAAM,kBAAmC;AAAA,UACvC;AAAA,UACA,SAAS,yDAAoB;AAAA,QAC/B;AAEA,YAAI,KAAK,SAAS,WAAW;AAC3B;AAAA,YACE,KAAK,YAAY,YACf,KAAK,YAAY,WACjB,KAAK,YAAY;AAAA,YACnB,gCAAgC,KAAK,OAAO;AAAA,UAC9C;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AACxD,cAAI,KAAK,YAAY,UAAU;AAC7B,iCACE,2CACC;AAAA,UACL;AAAA,QACF,WAAW,KAAK,SAAS,YAAY,KAAK,SAAS,YAAY;AAC7D,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D,OAAO;AACL,kBAAQ;AAAA,YACN,0BAA0B,KAAK,IAAI;AAAA,UACrC;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D;AAEA,eAAO,OAAO,MAAM,WAAW;AAC/B,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD,aAAK,OAAO,UAAU,2CAAqB,WAAU;AACrD;AAAA,MACF,SAAS,GAAQ;AACf,gCAAwB;AACxB,aAAK,SACH,uBAAG,aAAY,OAAO,MAAM,WAAW,IAAI;AAC7C,aAAK,aAAa,EAAE;AAEpB,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF;AAAA,IACF;AAGA,aAAS,IAAI,YAAY,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AACtD,WAAK,MAAM,CAAC,EAAE,SAAS;AAAA,IACzB;AAEA,QAAI,uBAAuB;AACzB,WAAK,SAAS;AAAA,IAChB,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAEA,QAAI,KAAK,MAAM,QAAQ;AAErB,YAAM,cAAc,KAAK,IAAI,WAAW,KAAK,MAAM,SAAS,CAAC;AAC7D,aAAO,KAAK,MAAM,WAAW,EAAE;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,iBAA0B;AACxB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,kBAAwC;AACtC,QAAI,KAAK,WAAW,SAAS;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,iBAAiB,KAAK,MAAM;AAAA,MAChC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,kBAAkB,GAAG;AACvB,aAAO,KAAK,MAAM,cAAc;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAsB;AACpB,UAAM,WAA0B;AAAA,MAC9B,YAAY,WAAW;AAAA,MACvB,YAAY,YAAY,mBAAmB,KAAK;AAAA,MAChD,SAAS,KAAK,IAAI;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,IACd;AACA,WAAO;AAAA,EACT;AACF;;;AC3MA,OAAOA,aAAY;;;ACAnB,OAAOA,aAAY;AAenB,SAAS,YAAY;AACrB,IAAM,aAAuB,CAAC;AAC9B,IAAM,gBAAwC,CAAC;AAExC,SAAS,gBACd,MACA,OACA,gBACQ;AACR,QAAM,SAAS,UAAU;AACzB,EAAAA,QAAO,QAAQ,+CAA+C;AAE9D,QAAM,KAAK,SAAS,KAAK;AACzB,MAAI,mBAAmB;AACvB,MAAI,qBAAqB,wBAAwB,GAAG;AAClD,uBAAmB;AAAA,EACrB,WAAW,qBAAqB,oBAAoB,GAAG;AACrD,uBAAmB;AAAA,EACrB;AAEA,QAAM,WAAqB;AAAA,IACzB,YAAY,WAAW;AAAA,IACvB,SAAS,KAAK,IAAI;AAAA,IAClB,YAAY,YAAY,mBAAmB,KAAK;AAAA,IAChD,mBAAmB;AAAA,EACrB;AACA,QAAM,YAAyB;AAAA,IAC7B,OAAO;AAAA,IACP,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAEA,mDAAiB;AAEjB,QAAM,aAAa,kBAAkB,WAAW,CAAC;AAEjD,MAAI,OAAO,cAAc,EAAE,MAAM,UAAU;AACzC,eAAW,cAAc,EAAE,CAAC,IAAI;AAAA,EAClC,OAAO;AACL,UAAM,SAAS,WAAW,KAAK,UAAU;AACzC,kBAAc,EAAE,IAAI,SAAS;AAAA,EAC/B;AAEA,SAAO;AACT;;;AD/BA,IAAqB,UAArB,MAGE;AAAA,EAaA,YACE,SAGA,KACA;AAbF,sBAAoD;AAclD,IAAAA,QAAO,SAAS,iCAAiC;AACjD,QAAI,OAAO,YAAY,YAAY;AACjC,WAAK,qBAAqB;AAAA,IAC5B,OAAO;AACL,WAAK,qBAAqB,MAAM,QAAQ,QAAQ,OAAO;AAAA,IACzD;AAEA,SAAK,kBAAkB,2BAAK;AAE5B,QAAI,QAAO,2BAAK,gBAAe,aAAa;AAC1C,WAAK,aAAa,IAAI;AAAA,IACxB;AACA,QAAI,QAAO,2BAAK,cAAa,aAAa;AACxC,WAAK,WAAW,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAMA,MAAM,OAAO,aAAqB,KAAkB;AAvEtD;AAwEI,UAAM,EAAE,OAAO,IAAI,OAAO,CAAC;AAC3B,IAAAA;AAAA,MACE,gBAAe,2BAAK;AAAA,MACpB;AAAA,IACF;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AACzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AAEtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,aAAa,aAAa,MAAM,IACnD,MAAM,iBAAiB;AAAA,MACrB,QAAQ,UAAU,KAAK;AAAA,MACvB;AAAA,MACA,0BAA0B;AAAA,MAC1B,aAAa,2BAAK;AAAA,IACpB,CAAC;AAEH,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,MACvC,gBAAgB,KAAK,UAAU,WAAW;AAAA,MAC1C;AAAA,IACF;AAEA,QAAI;AACJ,SAAI,iBAAY,WAAZ,mBAAoB,QAAQ;AAC9B,iBAAW;AAAA,EAAiC,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAC3E;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT,SAAS;AAAA,MACX;AAAA,MACA,aAAa,2BAAK;AAAA,MAClB,gBAAgB,CAAC;AAAA,MACjB,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,QAAQ,gBAAgB,UAAU,QAAW,cAAc;AAEjE,QAAI,UAAU;AACZ,cAAQ,MAAM,QAAQ;AACtB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,UAAM,WAA0B,CAAC;AACjC,gBAAY,SAAS,QAAQ,CAAC,SAAS;AACrC,UAAI,QAAQ,MAAM;AAChB,cAAM,UAAU,YAAY,KAAK,EAAE;AAEnC,YAAI,CAAC,SAAS;AACZ,kBAAQ;AAAA,YACN,kCAAkC,KAAK,EAAE;AAAA,UAC3C;AACA;AAAA,QACF;AACA,iBAAS,KAAK,OAAO;AAAA,MACvB;AAAA,IACF,CAAC;AAED;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI,SAAS,UAAU,GAAG;AACxB,cAAQ;AAAA,QACN,kEAAkE,WAAW;AAAA,MAC/E;AACA,aAAO,SAAS,CAAC;AAAA,IACnB;AACA,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO,SAAS,CAAC;AAAA,IACnB;AACA,WAAO;AAAA,EACT;AAAA,EAQA,MAAM,QAAW,YAA+C;AAvKlE;AAwKI,IAAAA;AAAA,MACE,OAAO,eAAe,YAAY,OAAO,eAAe;AAAA,MACxD,kDAAkD,OAAO,UAAU;AAAA,IACrE;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,SAAS;AAEvD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,IAAI,MAAM,qBAAwB;AAAA,MAC3D;AAAA,MACA,WAAW;AAAA,IACb,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,IACzC;AAEA,QAAI;AACJ,SAAI,iBAAY,WAAZ,mBAAoB,QAAQ;AAC9B,iBAAW;AAAA,EAAwB,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAClE;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AACA,UAAM,QAAQ,gBAAgB,UAAU,QAAW,cAAc;AAEjE,UAAM,EAAE,KAAK,IAAI;AACjB,QAAI,YAAY,CAAC,MAAM;AACrB,cAAQ,MAAM,QAAQ;AACtB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB,CAAC;AAAA,QACjB;AAAA,MACF;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,WAAsD;AACjE,QAAI,OAAO,cAAc,UAAU;AACjC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AACtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,aAAa,OAAO;AAAA,IAClD;AAEA,UAAM,EAAE,SAAS,KAAK,IAAI,aAAa;AACvC,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN;AAAA,MACA,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,eAAe;AAAA,MACf,kBAAkB;AAAA,MAClB,OAAO,OAAO,SAAY;AAAA,IAC5B;AACA,oBAAgB,UAAU,QAAW,cAAc;AAEnD,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AACF;;;AEvQA,IAAO,cAAQ","names":["assert"],"ignoreList":[],"sources":["../../src/ai-model/action-executor.ts","../../src/insight/index.ts","../../src/insight/utils.ts","../../src/index.ts"],"sourcesContent":["import assert from 'node:assert';\nimport { MIDSCENE_MODEL_NAME, getAIConfig } from '@/env';\nimport type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<any> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        task.timing.aiCost = (returnValue as any)?.aiCost || 0;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      return this.tasks[outputIndex].output;\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n","import assert from 'node:assert';\nimport { callAiFn } from '@/ai-model/common';\nimport { AiExtractElementInfo, AiInspectElement } from '@/ai-model/index';\nimport { AiAssert } from '@/ai-model/inspect';\nimport type {\n  AIElementResponse,\n  AISingleElementResponse,\n  BaseElement,\n  DumpSubscriber,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  PartialInsightDumpFromSDK,\n  UIContext,\n} from '@/types';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  callAI?: typeof callAiFn<AIElementResponse>;\n  quickAnswer?: Partial<AISingleElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  generateElement: InsightOptions['generateElement'];\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    this.generateElement = opt?.generateElement;\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    queryPrompt: string,\n    opt?: LocateOpts,\n  ): Promise<ElementType | null>;\n  async locate(queryPrompt: string, opt?: LocateOpts) {\n    const { callAI } = opt || {};\n    assert(\n      queryPrompt || opt?.quickAnswer,\n      'query or quickAnswer is required for locate',\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n    const context = await this.contextRetrieverFn('locate');\n\n    const startTime = Date.now();\n    const { parseResult, elementById, rawResponse, usage } =\n      await AiInspectElement({\n        callAI: callAI || this.aiVendorFn,\n        context,\n        targetElementDescription: queryPrompt,\n        quickAnswer: opt?.quickAnswer,\n      });\n    // const parseResult = await this.aiVendorFn<AIElementParseResponse>(msgs);\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `locate - AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      context,\n      userQuery: {\n        element: queryPrompt,\n      },\n      quickAnswer: opt?.quickAnswer,\n      matchedSection: [],\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const logId = emitInsightDump(dumpData, undefined, dumpSubscriber);\n\n    if (errorLog) {\n      console.error(errorLog);\n      throw new Error(errorLog);\n    }\n\n    const elements: BaseElement[] = [];\n    parseResult.elements.forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      logId,\n      dumpSubscriber,\n    );\n\n    if (elements.length >= 2) {\n      console.warn(\n        `locate: multiple elements found, return the first one. (query: ${queryPrompt})`,\n      );\n      return elements[0];\n    }\n    if (elements.length === 1) {\n      return elements[0];\n    }\n    return null;\n  }\n\n  async extract<T = any>(input: string): Promise<T>;\n  async extract<T extends Record<string, string>>(\n    input: T,\n  ): Promise<Record<keyof T, any>>;\n  async extract<T extends object>(input: Record<keyof T, string>): Promise<T>;\n\n  async extract<T>(dataDemand: InsightExtractParam): Promise<any> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      context,\n      userQuery: {\n        dataDemand,\n      },\n      matchedSection: [],\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n    const logId = emitInsightDump(dumpData, undefined, dumpSubscriber);\n\n    const { data } = parseResult;\n    if (errorLog && !data) {\n      console.error(errorLog);\n      throw new Error(errorLog);\n    }\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedSection: [],\n        data,\n      },\n      logId,\n      dumpSubscriber,\n    );\n\n    return {\n      data,\n      usage,\n    };\n  }\n\n  async assert(assertion: string): Promise<InsightAssertionResponse> {\n    if (typeof assertion !== 'string') {\n      throw new Error(\n        'This is the assert method for Midscene, the first argument should be a string. If you want to use the assert method from Node.js, please import it from the Node.js assert module.',\n      );\n    }\n\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      context,\n      userQuery: {\n        assertion,\n      },\n      matchedSection: [],\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, undefined, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n}\n","import assert from 'node:assert';\nimport {\n  MIDSCENE_MODEL_NAME,\n  MIDSCENE_USE_QWEN_VL,\n  MIDSCENE_USE_VLM_UI_TARS,\n  getAIConfig,\n  getAIConfigInBoolean,\n} from '@/env';\nimport type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getLogDir, getVersion, stringifyDumpData } from '@/utils';\nimport { uuid } from '@midscene/shared/utils';\nconst logContent: string[] = [];\nconst logIdIndexMap: Record<string, number> = {};\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  logId?: string,\n  dumpSubscriber?: DumpSubscriber,\n): string {\n  const logDir = getLogDir();\n  assert(logDir, 'logDir should be set before writing dump file');\n\n  const id = logId || uuid();\n  let modelDescription = '';\n  if (getAIConfigInBoolean(MIDSCENE_USE_VLM_UI_TARS)) {\n    modelDescription = 'vlm-ui-tars mode';\n  } else if (getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)) {\n    modelDescription = 'qwen-vl mode';\n  }\n\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n    model_description: modelDescription,\n  };\n  const finalData: InsightDump = {\n    logId: id,\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n\n  const dataString = stringifyDumpData(finalData, 2);\n\n  if (typeof logIdIndexMap[id] === 'number') {\n    logContent[logIdIndexMap[id]] = dataString;\n  } else {\n    const length = logContent.push(dataString);\n    logIdIndexMap[id] = length - 1;\n  }\n\n  return id;\n}\n","import { Executor } from './ai-model/action-executor';\nimport Insight from './insight';\nimport { getLogDirByType, getVersion, setLogDir } from './utils';\n\nexport {\n  plan,\n  transformElementPositionToId,\n  describeUserPage,\n  AiInspectElement,\n  AiAssert,\n} from './ai-model';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from './env';\n\nexport * from './types';\nexport default Insight;\nexport { Executor, setLogDir, getLogDirByType, Insight, getVersion };\n"]}

package/dist/lib/{chunk-P3XQH42C.js → chunk-XAXHXB3S.js}

@@ -188,7 +188,7 @@ function stringifyDumpData(data, indents) {
   return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-  return "0.11.4-beta-20250220160619.0";
+  return "0.12.0";
 }
 function debugLog(...message) {
   const debugMode = _chunkWVG2QNX6js.getAIConfig.call(void 0, _chunkWVG2QNX6js.MIDSCENE_DEBUG_MODE);
@@ -253,4 +253,4 @@ function uploadTestInfoToServer({ testUrl }) {
 
 exports.groupedActionDumpFileExt = groupedActionDumpFileExt; exports.getLogDir = getLogDir; exports.setLogDir = setLogDir; exports.getLogDirByType = getLogDirByType; exports.replaceStringWithFirstAppearance = replaceStringWithFirstAppearance; exports.reportHTMLContent = reportHTMLContent; exports.writeDumpReport = writeDumpReport; exports.writeLogFile = writeLogFile; exports.getTmpDir = getTmpDir; exports.getTmpFile = getTmpFile; exports.overlapped = overlapped; exports.sleep = sleep; exports.replacerForPageObject = replacerForPageObject; exports.stringifyDumpData = stringifyDumpData; exports.getVersion = getVersion; exports.uploadTestInfoToServer = uploadTestInfoToServer;
 
-//# sourceMappingURL=chunk-P3XQH42C.js.map
+//# sourceMappingURL=chunk-XAXHXB3S.js.map

package/dist/lib/{chunk-7ROPU2SY.js → chunk-YOJJI4TI.js}

@@ -1554,7 +1554,7 @@ You are a versatile professional in software UI automation. Your outstanding con
 - All the actions you composed MUST be based on the page context information you get.
 - Trust the "What have been done" field about the task (if any), don't repeat actions in it.
 - Respond only with valid JSON. Do not write an introduction or summary or markdown prefix like \`\`\`json\`\`\`.
-- If you cannot plan any action at all (i.e. empty actions array), set reason in the \`error\` field.
+- If the screenshot and the instruction are totally irrelevant, set reason in the \`error\` field.
 
 ## About the \`actions\` field
 
@@ -1701,7 +1701,7 @@ var planSchema = {
               },
               type: {
                 type: "string",
-                description: 'Type of action, like "Tap", "Hover", etc.'
+                description: 'Type of action, one of "Tap", "Hover" , "Input", "KeyboardPress", "Scroll", "ExpectedFalsyCondition", "Sleep"'
               },
               param: {
                 anyOf: [
@@ -1727,6 +1727,12 @@ var planSchema = {
                     },
                     required: ["direction", "scrollType", "distance"],
                     additionalProperties: false
+                  },
+                  {
+                    type: "object",
+                    properties: { reason: { type: "string" } },
+                    required: ["reason"],
+                    additionalProperties: false
                   }
                 ],
                 description: "Parameter of the action, can be null ONLY when the type field is Tap or Hover"
@@ -2647,4 +2653,4 @@ utf8/utf8.js:
   (*! https://mths.be/utf8js v3.0.0 by @mathias *)
 */
 
-//# sourceMappingURL=chunk-7ROPU2SY.js.map
+//# sourceMappingURL=chunk-YOJJI4TI.js.map