@aiscene/core 1.7.0

package/dist/lib/yaml/utils.js

@@ -0,0 +1,155 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.n = (module)=>{
+        var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
+        __webpack_require__.d(getter, {
+            a: getter
+        });
+        return getter;
+    };
+})();
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    buildDetailedLocateParam: ()=>buildDetailedLocateParam,
+    buildDetailedLocateParamAndRestParams: ()=>buildDetailedLocateParamAndRestParams,
+    parseYamlScript: ()=>parseYamlScript,
+    interpolateEnvVars: ()=>interpolateEnvVars
+});
+const logger_namespaceObject = require("@midscene/shared/logger");
+const utils_namespaceObject = require("@midscene/shared/utils");
+const external_js_yaml_namespaceObject = require("js-yaml");
+var external_js_yaml_default = /*#__PURE__*/ __webpack_require__.n(external_js_yaml_namespaceObject);
+const debugUtils = (0, logger_namespaceObject.getDebug)('yaml:utils');
+const multimodalLocateOptionFieldMap = {
+    images: true,
+    convertHttpImage2Base64: true
+};
+const multimodalLocateOptionKeys = Object.keys(multimodalLocateOptionFieldMap);
+function extractMultimodalPrompt(opt) {
+    if ('object' != typeof opt || null === opt) return;
+    const entries = multimodalLocateOptionKeys.map((key)=>[
+            key,
+            opt[key]
+        ]).filter(([, value])=>void 0 !== value);
+    return entries.length ? Object.fromEntries(entries) : void 0;
+}
+function interpolateEnvVars(content) {
+    const lines = content.split('\n');
+    const processedLines = lines.map((line)=>{
+        const trimmedLine = line.trimStart();
+        if (trimmedLine.startsWith('#')) return line;
+        return line.replace(/\$\{([^}]+)\}/g, (_, envVar)=>{
+            const value = process.env[envVar.trim()];
+            if (void 0 === value) throw new Error(`Environment variable "${envVar.trim()}" is not defined`);
+            return value;
+        });
+    });
+    return processedLines.join('\n');
+}
+function parseYamlScript(content, filePath) {
+    let processedContent = content;
+    if (-1 !== content.indexOf('android') && content.match(/deviceId:\s*(\d+)/)) {
+        let matchedDeviceId;
+        processedContent = content.replace(/deviceId:\s*(\d+)/g, (match, deviceId)=>{
+            matchedDeviceId = deviceId;
+            return `deviceId: '${deviceId}'`;
+        });
+        console.warn(`please use string-style deviceId in yaml script, for example: deviceId: "${matchedDeviceId}"`);
+    }
+    const interpolatedContent = interpolateEnvVars(processedContent);
+    const obj = external_js_yaml_default().load(interpolatedContent, {
+        schema: external_js_yaml_default().JSON_SCHEMA
+    });
+    const pathTip = filePath ? `, failed to load ${filePath}` : '';
+    (0, utils_namespaceObject.assert)(obj.tasks, `property "tasks" is required in yaml script ${pathTip}`);
+    (0, utils_namespaceObject.assert)(Array.isArray(obj.tasks), `property "tasks" must be an array in yaml script, but got ${obj.tasks}`);
+    return obj;
+}
+function buildDetailedLocateParam(locatePrompt, opt) {
+    debugUtils('will call buildDetailedLocateParam', locatePrompt, opt);
+    let normalizedLocatePrompt = locatePrompt;
+    if ('object' == typeof locatePrompt && null !== locatePrompt && 'prompt' in locatePrompt) {
+        const { prompt: innerPrompt, ...rest } = locatePrompt;
+        const hasMultimodalFields = Object.keys(rest).length > 0;
+        normalizedLocatePrompt = hasMultimodalFields ? locatePrompt : innerPrompt;
+    }
+    let prompt = normalizedLocatePrompt || opt?.prompt || opt?.locate;
+    let deepLocate = false;
+    let cacheable = true;
+    let xpath;
+    if ('object' == typeof opt && null !== opt) {
+        deepLocate = opt.deepLocate ?? opt.deepThink ?? false;
+        cacheable = opt.cacheable ?? true;
+        xpath = opt.xpath;
+        if (locatePrompt && opt.prompt && locatePrompt !== opt.prompt) console.warn('conflict prompt for item', locatePrompt, opt, 'maybe you put the prompt in the wrong place');
+        prompt = prompt || opt.prompt;
+    }
+    if (!prompt) return void debugUtils('no prompt, will return undefined in buildDetailedLocateParam', opt);
+    const multimodalPrompt = extractMultimodalPrompt(opt);
+    if (multimodalPrompt) prompt = 'string' == typeof prompt ? {
+        prompt,
+        ...multimodalPrompt
+    } : {
+        ...prompt,
+        ...multimodalPrompt
+    };
+    return {
+        prompt,
+        deepLocate,
+        cacheable,
+        xpath
+    };
+}
+function buildDetailedLocateParamAndRestParams(locatePrompt, opt, excludeKeys = []) {
+    const multimodalPrompt = extractMultimodalPrompt(opt);
+    const locateParam = buildDetailedLocateParam(locatePrompt, opt);
+    const restParams = {};
+    if ('object' == typeof opt && null !== opt) {
+        const allKeys = Object.keys(opt);
+        const locateParamKeys = Object.keys(locateParam || {});
+        const multimodalPromptKeys = 'object' == typeof locateParam?.prompt && locateParam?.prompt !== null ? Object.keys(multimodalPrompt || {}) : [];
+        for (const key of allKeys)if (!locateParamKeys.includes(key) && !multimodalPromptKeys.includes(key) && !excludeKeys.includes(key) && 'locate' !== key) restParams[key] = opt[key];
+    }
+    return {
+        locateParam,
+        restParams
+    };
+}
+exports.buildDetailedLocateParam = __webpack_exports__.buildDetailedLocateParam;
+exports.buildDetailedLocateParamAndRestParams = __webpack_exports__.buildDetailedLocateParamAndRestParams;
+exports.interpolateEnvVars = __webpack_exports__.interpolateEnvVars;
+exports.parseYamlScript = __webpack_exports__.parseYamlScript;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "buildDetailedLocateParam",
+    "buildDetailedLocateParamAndRestParams",
+    "interpolateEnvVars",
+    "parseYamlScript"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=utils.js.map

package/dist/lib/yaml/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"yaml/utils.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/yaml/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type {\n  DetailedLocateParam,\n  LocateOption,\n  MidsceneYamlScript,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\n\nconst debugUtils = getDebug('yaml:utils');\n\nconst multimodalLocateOptionFieldMap: Record<keyof TMultimodalPrompt, true> = {\n  images: true,\n  convertHttpImage2Base64: true,\n};\n\nconst multimodalLocateOptionKeys = Object.keys(\n  multimodalLocateOptionFieldMap,\n) as Array<keyof TMultimodalPrompt>;\n\nfunction extractMultimodalPrompt(\n  opt?: LocateOption,\n): Partial<TMultimodalPrompt> | undefined {\n  if (typeof opt !== 'object' || opt === null) {\n    return undefined;\n  }\n\n  const entries = multimodalLocateOptionKeys\n    .map((key) => [key, opt[key]] as const)\n    .filter(([, value]) => value !== undefined);\n\n  return entries.length\n    ? (Object.fromEntries(entries) as Partial<TMultimodalPrompt>)\n    : undefined;\n}\n\nexport function interpolateEnvVars(content: string): string {\n  // Process line by line to skip commented lines\n  const lines = content.split('\\n');\n  const processedLines = lines.map((line) => {\n    // Check if the line is a YAML comment (starts with # after optional whitespace)\n    const trimmedLine = line.trimStart();\n    if (trimmedLine.startsWith('#')) {\n      // Skip interpolation for comment lines\n      return line;\n    }\n\n    // Process environment variables for non-comment lines\n    return line.replace(/\\$\\{([^}]+)\\}/g, (_, envVar) => {\n      const value = process.env[envVar.trim()];\n      if (value === undefined) {\n        throw new Error(\n          `Environment variable \"${envVar.trim()}\" is not defined`,\n        );\n      }\n      return value;\n    });\n  });\n\n  return processedLines.join('\\n');\n}\n\nexport function parseYamlScript(\n  content: string,\n  filePath?: string,\n): MidsceneYamlScript {\n  let processedContent = content;\n  if (content.indexOf('android') !== -1 && content.match(/deviceId:\\s*(\\d+)/)) {\n    let matchedDeviceId;\n    processedContent = content.replace(\n      /deviceId:\\s*(\\d+)/g,\n      (match, deviceId) => {\n        matchedDeviceId = deviceId;\n        return `deviceId: '${deviceId}'`;\n      },\n    );\n    console.warn(\n      `please use string-style deviceId in yaml script, for example: deviceId: \"${matchedDeviceId}\"`,\n    );\n  }\n  const interpolatedContent = interpolateEnvVars(processedContent);\n  const obj = yaml.load(interpolatedContent, {\n    schema: yaml.JSON_SCHEMA,\n  }) as MidsceneYamlScript;\n\n  const pathTip = filePath ? `, failed to load ${filePath}` : '';\n  assert(obj.tasks, `property \"tasks\" is required in yaml script ${pathTip}`);\n  assert(\n    Array.isArray(obj.tasks),\n    `property \"tasks\" must be an array in yaml script, but got ${obj.tasks}`,\n  );\n  return obj;\n}\n\nexport function buildDetailedLocateParam(\n  locatePrompt: TUserPrompt,\n  opt?: LocateOption,\n): DetailedLocateParam | undefined {\n  debugUtils('will call buildDetailedLocateParam', locatePrompt, opt);\n  // Normalize object-form TUserPrompt: when the object only contains a\n  // `prompt` string (no multimodal fields like `images`), unwrap it to\n  // avoid double nesting like { prompt: { prompt: '...' } }.\n  let normalizedLocatePrompt: TUserPrompt = locatePrompt;\n  if (\n    typeof locatePrompt === 'object' &&\n    locatePrompt !== null &&\n    'prompt' in locatePrompt\n  ) {\n    const { prompt: innerPrompt, ...rest } = locatePrompt;\n    const hasMultimodalFields = Object.keys(rest).length > 0;\n    normalizedLocatePrompt = hasMultimodalFields ? locatePrompt : innerPrompt;\n  }\n\n  let prompt = normalizedLocatePrompt || opt?.prompt || (opt as any)?.locate; // as a shortcut\n  let deepLocate = false;\n  let cacheable = true;\n  let xpath = undefined;\n\n  if (typeof opt === 'object' && opt !== null) {\n    // Backward-compatible: accept `deepThink` as a deprecated alias for `deepLocate`.\n    // All downstream code works on `deepLocate` only; the compatibility resolution\n    // is intentionally kept here at the entry point so it does not bleed through\n    // the rest of the call stack.\n    deepLocate = opt.deepLocate ?? opt.deepThink ?? false;\n    cacheable = opt.cacheable ?? true;\n    xpath = opt.xpath;\n    if (locatePrompt && opt.prompt && locatePrompt !== opt.prompt) {\n      console.warn(\n        'conflict prompt for item',\n        locatePrompt,\n        opt,\n        'maybe you put the prompt in the wrong place',\n      );\n    }\n    prompt = prompt || opt.prompt;\n  }\n\n  if (!prompt) {\n    debugUtils(\n      'no prompt, will return undefined in buildDetailedLocateParam',\n      opt,\n    );\n    return undefined;\n  }\n\n  const multimodalPrompt = extractMultimodalPrompt(opt);\n  if (multimodalPrompt) {\n    prompt =\n      typeof prompt === 'string'\n        ? {\n            prompt,\n            ...multimodalPrompt,\n          }\n        : {\n            ...prompt,\n            ...multimodalPrompt,\n          };\n  }\n\n  return {\n    prompt,\n    deepLocate,\n    cacheable,\n    xpath,\n  };\n}\n\nexport function buildDetailedLocateParamAndRestParams(\n  locatePrompt: TUserPrompt,\n  opt: LocateOption | undefined,\n  excludeKeys: string[] = [],\n): {\n  locateParam: DetailedLocateParam | undefined;\n  restParams: Record<string, any>;\n} {\n  const multimodalPrompt = extractMultimodalPrompt(opt);\n  const locateParam = buildDetailedLocateParam(locatePrompt, opt);\n\n  // Extract all keys from opt except the ones already included in locateParam\n  const restParams: Record<string, any> = {};\n\n  if (typeof opt === 'object' && opt !== null) {\n    // Get all keys from opt\n    const allKeys = Object.keys(opt);\n\n    // Keys already included in locateParam: prompt, deepLocate, cacheable, xpath\n    const locateParamKeys = Object.keys(locateParam || {});\n    const multimodalPromptKeys =\n      typeof locateParam?.prompt === 'object' && locateParam?.prompt !== null\n        ? Object.keys(multimodalPrompt || {})\n        : [];\n\n    // Extract all other keys\n    for (const key of allKeys) {\n      if (\n        !locateParamKeys.includes(key) &&\n        !multimodalPromptKeys.includes(key) &&\n        !excludeKeys.includes(key) &&\n        key !== 'locate'\n      ) {\n        restParams[key] = opt[key as keyof LocateOption];\n      }\n    }\n  }\n\n  return {\n    locateParam,\n    restParams,\n  };\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","debugUtils","getDebug","multimodalLocateOptionFieldMap","multimodalLocateOptionKeys","extractMultimodalPrompt","opt","entries","value","undefined","interpolateEnvVars","content","lines","processedLines","line","trimmedLine","_","envVar","process","Error","parseYamlScript","filePath","processedContent","matchedDeviceId","match","deviceId","console","interpolatedContent","yaml","pathTip","assert","Array","buildDetailedLocateParam","locatePrompt","normalizedLocatePrompt","innerPrompt","rest","hasMultimodalFields","prompt","deepLocate","cacheable","xpath","multimodalPrompt","buildDetailedLocateParamAndRestParams","excludeKeys","locateParam","restParams","allKeys","locateParamKeys","multimodalPromptKeys"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACIA,MAAMI,aAAaC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAE5B,MAAMC,iCAAwE;IAC5E,QAAQ;IACR,yBAAyB;AAC3B;AAEA,MAAMC,6BAA6BP,OAAO,IAAI,CAC5CM;AAGF,SAASE,wBACPC,GAAkB;IAElB,IAAI,AAAe,YAAf,OAAOA,OAAoBA,AAAQ,SAARA,KAC7B;IAGF,MAAMC,UAAUH,2BACb,GAAG,CAAC,CAACR,MAAQ;YAACA;YAAKU,GAAG,CAACV,IAAI;SAAC,EAC5B,MAAM,CAAC,CAAC,GAAGY,MAAM,GAAKA,AAAUC,WAAVD;IAEzB,OAAOD,QAAQ,MAAM,GAChBV,OAAO,WAAW,CAACU,WACpBE;AACN;AAEO,SAASC,mBAAmBC,OAAe;IAEhD,MAAMC,QAAQD,QAAQ,KAAK,CAAC;IAC5B,MAAME,iBAAiBD,MAAM,GAAG,CAAC,CAACE;QAEhC,MAAMC,cAAcD,KAAK,SAAS;QAClC,IAAIC,YAAY,UAAU,CAAC,MAEzB,OAAOD;QAIT,OAAOA,KAAK,OAAO,CAAC,kBAAkB,CAACE,GAAGC;YACxC,MAAMT,QAAQU,QAAQ,GAAG,CAACD,OAAO,IAAI,GAAG;YACxC,IAAIT,AAAUC,WAAVD,OACF,MAAM,IAAIW,MACR,CAAC,sBAAsB,EAAEF,OAAO,IAAI,GAAG,gBAAgB,CAAC;YAG5D,OAAOT;QACT;IACF;IAEA,OAAOK,eAAe,IAAI,CAAC;AAC7B;AAEO,SAASO,gBACdT,OAAe,EACfU,QAAiB;IAEjB,IAAIC,mBAAmBX;IACvB,IAAIA,AAA+B,OAA/BA,QAAQ,OAAO,CAAC,cAAqBA,QAAQ,KAAK,CAAC,sBAAsB;QAC3E,IAAIY;QACJD,mBAAmBX,QAAQ,OAAO,CAChC,sBACA,CAACa,OAAOC;YACNF,kBAAkBE;YAClB,OAAO,CAAC,WAAW,EAAEA,SAAS,CAAC,CAAC;QAClC;QAEFC,QAAQ,IAAI,CACV,CAAC,yEAAyE,EAAEH,gBAAgB,CAAC,CAAC;IAElG;IACA,MAAMI,sBAAsBjB,mBAAmBY;IAC/C,MAAMxB,MAAM8B,2BAAAA,IAAS,CAACD,qBAAqB;QACzC,QAAQC,AAAAA,2BAAAA,WAAgB;IAC1B;IAEA,MAAMC,UAAUR,WAAW,CAAC,iBAAiB,EAAEA,UAAU,GAAG;IAC5DS,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOhC,IAAI,KAAK,EAAE,CAAC,4CAA4C,EAAE+B,SAAS;IAC1EC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEC,MAAM,OAAO,CAACjC,IAAI,KAAK,GACvB,CAAC,0DAA0D,EAAEA,IAAI,KAAK,EAAE;IAE1E,OAAOA;AACT;AAEO,SAASkC,yBACdC,YAAyB,EACzB3B,GAAkB;IAElBL,WAAW,sCAAsCgC,cAAc3B;IAI/D,IAAI4B,yBAAsCD;IAC1C,IACE,AAAwB,YAAxB,OAAOA,gBACPA,AAAiB,SAAjBA,gBACA,YAAYA,cACZ;QACA,MAAM,EAAE,QAAQE,WAAW,EAAE,GAAGC,MAAM,GAAGH;QACzC,MAAMI,sBAAsBxC,OAAO,IAAI,CAACuC,MAAM,MAAM,GAAG;QACvDF,yBAAyBG,sBAAsBJ,eAAeE;IAChE;IAEA,IAAIG,SAASJ,0BAA0B5B,KAAK,UAAWA,KAAa;IACpE,IAAIiC,aAAa;IACjB,IAAIC,YAAY;IAChB,IAAIC;IAEJ,IAAI,AAAe,YAAf,OAAOnC,OAAoBA,AAAQ,SAARA,KAAc;QAK3CiC,aAAajC,IAAI,UAAU,IAAIA,IAAI,SAAS,IAAI;QAChDkC,YAAYlC,IAAI,SAAS,IAAI;QAC7BmC,QAAQnC,IAAI,KAAK;QACjB,IAAI2B,gBAAgB3B,IAAI,MAAM,IAAI2B,iBAAiB3B,IAAI,MAAM,EAC3DoB,QAAQ,IAAI,CACV,4BACAO,cACA3B,KACA;QAGJgC,SAASA,UAAUhC,IAAI,MAAM;IAC/B;IAEA,IAAI,CAACgC,QAAQ,YACXrC,WACE,gEACAK;IAKJ,MAAMoC,mBAAmBrC,wBAAwBC;IACjD,IAAIoC,kBACFJ,SACE,AAAkB,YAAlB,OAAOA,SACH;QACEA;QACA,GAAGI,gBAAgB;IACrB,IACA;QACE,GAAGJ,MAAM;QACT,GAAGI,gBAAgB;IACrB;IAGR,OAAO;QACLJ;QACAC;QACAC;QACAC;IACF;AACF;AAEO,SAASE,sCACdV,YAAyB,EACzB3B,GAA6B,EAC7BsC,cAAwB,EAAE;IAK1B,MAAMF,mBAAmBrC,wBAAwBC;IACjD,MAAMuC,cAAcb,yBAAyBC,cAAc3B;IAG3D,MAAMwC,aAAkC,CAAC;IAEzC,IAAI,AAAe,YAAf,OAAOxC,OAAoBA,AAAQ,SAARA,KAAc;QAE3C,MAAMyC,UAAUlD,OAAO,IAAI,CAACS;QAG5B,MAAM0C,kBAAkBnD,OAAO,IAAI,CAACgD,eAAe,CAAC;QACpD,MAAMI,uBACJ,AAA+B,YAA/B,OAAOJ,aAAa,UAAuBA,aAAa,WAAW,OAC/DhD,OAAO,IAAI,CAAC6C,oBAAoB,CAAC,KACjC,EAAE;QAGR,KAAK,MAAM9C,OAAOmD,QAChB,IACE,CAACC,gBAAgB,QAAQ,CAACpD,QAC1B,CAACqD,qBAAqB,QAAQ,CAACrD,QAC/B,CAACgD,YAAY,QAAQ,CAAChD,QACtBA,AAAQ,aAARA,KAEAkD,UAAU,CAAClD,IAAI,GAAGU,GAAG,CAACV,IAA0B;IAGtD;IAEA,OAAO;QACLiD;QACAC;IACF;AACF"}

package/dist/lib/yaml.js

@@ -0,0 +1,20 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+for(var __rspack_i in __webpack_exports__)exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=yaml.js.map

package/dist/lib/yaml.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"yaml.js","sources":["webpack/runtime/make_namespace_object"],"sourcesContent":["// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","Symbol","Object"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOC,UAA0BA,OAAO,WAAW,EACrDC,OAAO,cAAc,CAAC,UAASD,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEC,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/types/agent/agent.d.ts

@@ -0,0 +1,205 @@
+import type { TUserPrompt } from '../ai-model/index';
+import Service from '../service/index';
+import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, GroupedActionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type UIContext } from '../types';
+export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
+import type { AbstractInterface } from '../device';
+import type { TaskRunner } from '../task-runner';
+import { ModelConfigManager } from '@midscene/shared/env';
+import { TaskCache } from './task-cache';
+import { TaskExecutor } from './tasks';
+export type AiActOptions = {
+    cacheable?: boolean;
+    fileChooserAccept?: string | string[];
+    deepThink?: DeepThinkOption;
+    deepLocate?: boolean;
+    abortSignal?: AbortSignal;
+};
+export declare class Agent<InterfaceType extends AbstractInterface = AbstractInterface> {
+    interface: InterfaceType;
+    service: Service;
+    dump: GroupedActionDump;
+    reportFile?: string | null;
+    reportFileName?: string;
+    taskExecutor: TaskExecutor;
+    opts: AgentOpt;
+    /**
+     * If true, the agent will not perform any actions
+     */
+    dryMode: boolean;
+    onTaskStartTip?: OnTaskStartTip;
+    taskCache?: TaskCache;
+    private dumpUpdateListeners;
+    get onDumpUpdate(): ((dump: string, executionDump?: ExecutionDump) => void) | undefined;
+    set onDumpUpdate(callback: ((dump: string, executionDump?: ExecutionDump) => void) | undefined);
+    destroyed: boolean;
+    modelConfigManager: ModelConfigManager;
+    /**
+     * Frozen page context for consistent AI operations
+     */
+    private frozenUIContext?;
+    private get aiActContext();
+    /**
+     * Flag to track if VL model warning has been shown
+     */
+    private hasWarnedNonVLModel;
+    private executionDumpIndexByRunner;
+    private fullActionSpace;
+    private reportGenerator;
+    get page(): InterfaceType;
+    /**
+     * Ensures VL model warning is shown once when needed
+     */
+    private ensureVLModelWarning;
+    private resolveReplanningCycleLimit;
+    constructor(interfaceInstance: InterfaceType, opts?: AgentOpt);
+    getActionSpace(): Promise<DeviceAction[]>;
+    private static readonly CONTEXT_RETRY_MAX;
+    private static readonly CONTEXT_RETRY_DELAY_MS;
+    /**
+     * Override in subclasses to indicate which errors are transient and should
+     * trigger an automatic retry when building the UI context.
+     * Returns `false` by default (no retry).
+     */
+    protected isRetryableContextError(_error: unknown): boolean;
+    getUIContext(action?: ServiceAction): Promise<UIContext>;
+    _snapshotContext(): Promise<UIContext>;
+    /**
+     * @deprecated Use {@link setAIActContext} instead.
+     */
+    setAIActionContext(prompt: string): Promise<void>;
+    setAIActContext(prompt: string): Promise<void>;
+    resetDump(): GroupedActionDump;
+    appendExecutionDump(execution: ExecutionDump, runner?: TaskRunner): void;
+    dumpDataString(opt?: {
+        inlineScreenshots?: boolean;
+    }): string;
+    reportHTMLString(opt?: {
+        inlineScreenshots?: boolean;
+    }): string;
+    private lastExecutionDump?;
+    writeOutActionDumps(executionDump?: ExecutionDump): void;
+    private getGroupMeta;
+    private callbackOnTaskStartTip;
+    wrapActionInActionSpace<T extends DeviceAction>(name: string): (param: ActionParam<T>) => Promise<ActionReturn<T>>;
+    callActionInActionSpace<T = any>(type: string, opt?: T): Promise<any>;
+    aiTap(locatePrompt: TUserPrompt, opt?: LocateOption & {
+        fileChooserAccept?: string | string[];
+    }): Promise<any>;
+    aiRightClick(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiDoubleClick(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiHover(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiInput(locatePrompt: TUserPrompt, opt: LocateOption & {
+        value: string | number;
+    } & {
+        autoDismissKeyboard?: boolean;
+    } & {
+        mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
+    }): Promise<any>;
+    /**
+     * @deprecated Use aiInput(locatePrompt, opt) instead where opt contains the value
+     */
+    aiInput(value: string | number, locatePrompt: TUserPrompt, opt?: LocateOption & {
+        autoDismissKeyboard?: boolean;
+    } & {
+        mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
+    }): Promise<any>;
+    aiKeyboardPress(locatePrompt: TUserPrompt, opt: LocateOption & {
+        keyName: string;
+    }): Promise<any>;
+    /**
+     * @deprecated Use aiKeyboardPress(locatePrompt, opt) instead where opt contains the keyName
+     */
+    aiKeyboardPress(keyName: string, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiScroll(locatePrompt: TUserPrompt | undefined, opt: LocateOption & ScrollParam): Promise<any>;
+    /**
+     * @deprecated Use aiScroll(locatePrompt, opt) instead where opt contains the scroll parameters
+     */
+    aiScroll(scrollParam: ScrollParam, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiPinch(locatePrompt: TUserPrompt | undefined, opt: LocateOption & {
+        direction: 'in' | 'out';
+        distance?: number;
+        duration?: number;
+    }): Promise<any>;
+    aiAct(taskPrompt: string, opt?: AiActOptions): Promise<string | undefined>;
+    /**
+     * @deprecated Use {@link Agent.aiAct} instead.
+     */
+    aiAction(taskPrompt: string, opt?: AiActOptions): Promise<string | undefined>;
+    aiQuery<ReturnType = any>(demand: ServiceExtractParam, opt?: ServiceExtractOption): Promise<ReturnType>;
+    aiBoolean(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<boolean>;
+    aiNumber(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<number>;
+    aiString(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<string>;
+    aiAsk(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<string>;
+    describeElementAtPoint(center: [number, number], opt?: {
+        verifyPrompt?: boolean;
+        retryLimit?: number;
+        deepLocate?: boolean;
+    } & LocatorValidatorOption): Promise<AgentDescribeElementAtPointResult>;
+    verifyLocator(prompt: string, locateOpt: LocateOption | undefined, expectCenter: [number, number], verifyLocateOption?: LocatorValidatorOption): Promise<LocateValidatorResult>;
+    aiLocate(prompt: TUserPrompt, opt?: LocateOption): Promise<Pick<LocateResultElement, "rect" | "center">>;
+    aiAssert(assertion: TUserPrompt, msg?: string, opt?: AgentAssertOpt & ServiceExtractOption): Promise<{
+        pass: boolean;
+        thought: string | undefined;
+        message: string | undefined;
+    } | undefined>;
+    aiWaitFor(assertion: TUserPrompt, opt?: AgentWaitForOpt): Promise<void>;
+    ai(...args: Parameters<typeof this.aiAct>): Promise<string | undefined>;
+    runYaml(yamlScriptContent: string): Promise<{
+        result: Record<string, any>;
+    }>;
+    evaluateJavaScript(script: string): Promise<any>;
+    /**
+     * Add a dump update listener
+     * @param listener Listener function
+     * @returns A remove function that can be called to remove this listener
+     */
+    addDumpUpdateListener(listener: (dump: string, executionDump?: ExecutionDump) => void): () => void;
+    /**
+     * Remove a dump update listener
+     * @param listener The listener function to remove
+     */
+    removeDumpUpdateListener(listener: (dump: string, executionDump?: ExecutionDump) => void): void;
+    /**
+     * Clear all dump update listeners
+     */
+    clearDumpUpdateListeners(): void;
+    destroy(): Promise<void>;
+    recordToReport(title?: string, opt?: {
+        content: string;
+    }): Promise<void>;
+    /**
+     * @deprecated Use {@link Agent.recordToReport} instead.
+     */
+    logScreenshot(title?: string, opt?: {
+        content: string;
+    }): Promise<void>;
+    _unstableLogContent(): {
+        groupName: string;
+        groupDescription: string | undefined;
+        executions: ExecutionDump[];
+    };
+    /**
+     * Freezes the current page context to be reused in subsequent AI operations
+     * This avoids recalculating page context for each operation
+     */
+    freezePageContext(): Promise<void>;
+    /**
+     * Unfreezes the page context, allowing AI operations to calculate context dynamically
+     */
+    unfreezePageContext(): Promise<void>;
+    /**
+     * Process cache configuration and return normalized cache settings
+     */
+    private processCacheConfig;
+    private normalizeFilePaths;
+    private normalizeFileInput;
+    /**
+     * Manually flush cache to file
+     * @param options - Optional configuration
+     * @param options.cleanUnused - If true, removes unused cache records before flushing
+     */
+    flushCache(options?: {
+        cleanUnused?: boolean;
+    }): Promise<void>;
+}
+export declare const createAgent: (interfaceInstance: AbstractInterface, opts?: AgentOpt) => Agent<AbstractInterface>;

package/dist/types/agent/execution-session.d.ts

@@ -0,0 +1,36 @@
+import { type TaskExecutionError, TaskRunner } from '../task-runner';
+import type { ExecutionTaskApply, ExecutionTaskProgressOptions, UIContext } from '../types';
+type ExecutionSessionOptions = ExecutionTaskProgressOptions & {
+    tasks?: ExecutionTaskApply[];
+    onTaskUpdate?: (runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void;
+};
+/**
+ * Thin wrapper around {@link TaskRunner} that represents a single linear execution run.
+ */
+export declare class ExecutionSession {
+    private readonly runner;
+    constructor(name: string, contextProvider: () => Promise<UIContext>, options?: ExecutionSessionOptions);
+    append(tasks: ExecutionTaskApply[] | ExecutionTaskApply, options?: {
+        allowWhenError?: boolean;
+    }): Promise<void>;
+    appendAndRun(tasks: ExecutionTaskApply[] | ExecutionTaskApply, options?: {
+        allowWhenError?: boolean;
+    }): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
+    run(options?: {
+        allowWhenError?: boolean;
+    }): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
+    isInErrorState(): boolean;
+    latestErrorTask(): import("../types").ExecutionTask | null;
+    appendErrorPlan(errorMsg: string): Promise<{
+        output: undefined;
+        runner: TaskRunner;
+    }>;
+    getRunner(): TaskRunner;
+}
+export {};

package/dist/types/agent/index.d.ts

@@ -0,0 +1,10 @@
+export { Agent, createAgent } from './agent';
+export { commonContextParser } from './utils';
+export { getReportFileName, printReportMsg, } from './utils';
+export { extractInsightParam, locateParamStr, paramStr, taskTitleStr, typeStr, } from './ui-utils';
+export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
+export { cacheFileExt } from './task-cache';
+export { TaskExecutor } from './tasks';
+export { getCurrentExecutionFile } from './utils';
+export type { AgentOpt } from '../types';
+export type { AiActOptions } from './agent';

package/dist/types/agent/task-builder.d.ts

@@ -0,0 +1,34 @@
+import type { AbstractInterface } from '../device';
+import type Service from '../service';
+import type { DetailedLocateParam, DeviceAction, ExecutionTaskApply, PlanningAction, PlanningLocateParam } from '../types';
+import type { IModelConfig } from '@midscene/shared/env';
+import type { TaskCache } from './task-cache';
+export declare function locatePlanForLocate(param: string | DetailedLocateParam): PlanningAction<PlanningLocateParam>;
+interface TaskBuilderDeps {
+    interfaceInstance: AbstractInterface;
+    service: Service;
+    taskCache?: TaskCache;
+    actionSpace: DeviceAction[];
+    waitAfterAction?: number;
+}
+interface BuildOptions {
+    cacheable?: boolean;
+    deepLocate?: boolean;
+    abortSignal?: AbortSignal;
+}
+export declare class TaskBuilder {
+    private readonly interface;
+    private readonly service;
+    private readonly taskCache?;
+    private readonly actionSpace;
+    private readonly waitAfterAction?;
+    constructor({ interfaceInstance, service, taskCache, actionSpace, waitAfterAction, }: TaskBuilderDeps);
+    build(plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, options?: BuildOptions): Promise<{
+        tasks: ExecutionTaskApply[];
+    }>;
+    private handleFinishedPlan;
+    private handleLocatePlan;
+    private handleActionPlan;
+    private createLocateTask;
+}
+export {};

package/dist/types/agent/task-cache.d.ts

@@ -0,0 +1,49 @@
+import type { TUserPrompt } from '../ai-model';
+import type { ElementCacheFeature } from '../types';
+export declare const debug: import("@midscene/shared/logger").DebugFunction;
+export interface PlanningCache {
+    type: 'plan';
+    prompt: string;
+    yamlWorkflow: string;
+}
+export interface LocateCache {
+    type: 'locate';
+    prompt: TUserPrompt;
+    cache?: ElementCacheFeature;
+    /** @deprecated kept for backward compatibility */
+    xpaths?: string[];
+}
+export interface MatchCacheResult<T extends PlanningCache | LocateCache> {
+    cacheContent: T;
+    cacheUsable: boolean;
+    updateFn: (cb: (cache: T) => void) => void;
+}
+export type CacheFileContent = {
+    midsceneVersion: string;
+    cacheId: string;
+    caches: Array<PlanningCache | LocateCache>;
+};
+export declare const cacheFileExt = ".cache.yaml";
+export declare class TaskCache {
+    cacheId: string;
+    cacheFilePath?: string;
+    cache: CacheFileContent;
+    isCacheResultUsed: boolean;
+    cacheOriginalLength: number;
+    readOnlyMode: boolean;
+    writeOnlyMode: boolean;
+    private matchedCacheIndices;
+    constructor(cacheId: string, isCacheResultUsed: boolean, cacheFilePath?: string, options?: {
+        readOnly?: boolean;
+        writeOnly?: boolean;
+    });
+    matchCache(prompt: TUserPrompt, type: 'plan' | 'locate'): MatchCacheResult<PlanningCache | LocateCache> | undefined;
+    matchPlanCache(prompt: string): MatchCacheResult<PlanningCache> | undefined;
+    matchLocateCache(prompt: TUserPrompt): MatchCacheResult<LocateCache> | undefined;
+    appendCache(cache: PlanningCache | LocateCache): void;
+    loadCacheFromFile(): CacheFileContent | undefined;
+    flushCacheToFile(options?: {
+        cleanUnused?: boolean;
+    }): void;
+    updateOrAppendCacheRecord(newRecord: PlanningCache | LocateCache, cachedRecord?: MatchCacheResult<PlanningCache | LocateCache>): void;
+}

package/dist/types/agent/tasks.d.ts

@@ -0,0 +1,69 @@
+import { type TMultimodalPrompt, type TUserPrompt } from '../common';
+import type { AbstractInterface } from '../device';
+import type Service from '../service';
+import type { TaskRunner } from '../task-runner';
+import { TaskExecutionError } from '../task-runner';
+import type { DeepThinkOption, DeviceAction, ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam } from '../types';
+import { type IModelConfig } from '@midscene/shared/env';
+import type { TaskCache } from './task-cache';
+export { locatePlanForLocate } from './task-builder';
+interface ExecutionResult<OutputType = any> {
+    output: OutputType;
+    thought?: string;
+    runner: TaskRunner;
+}
+interface TaskExecutorHooks {
+    onTaskUpdate?: (runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void;
+}
+export { TaskExecutionError };
+export declare class TaskExecutor {
+    interface: AbstractInterface;
+    service: Service;
+    taskCache?: TaskCache;
+    private readonly providedActionSpace;
+    private readonly taskBuilder;
+    onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];
+    private readonly hooks?;
+    replanningCycleLimit?: number;
+    waitAfterAction?: number;
+    useDeviceTimestamp?: boolean;
+    get page(): AbstractInterface;
+    constructor(interfaceInstance: AbstractInterface, service: Service, opts: {
+        taskCache?: TaskCache;
+        onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
+        replanningCycleLimit?: number;
+        waitAfterAction?: number;
+        useDeviceTimestamp?: boolean;
+        hooks?: TaskExecutorHooks;
+        actionSpace: DeviceAction[];
+    });
+    private createExecutionSession;
+    private getActionSpace;
+    /**
+     * Get a readable time string using device time when configured.
+     * This method respects the useDeviceTimestamp configuration.
+     * @param format - Optional format string
+     * @returns A formatted time string
+     */
+    private getTimeString;
+    convertPlanToExecutable(plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, options?: {
+        cacheable?: boolean;
+        deepLocate?: boolean;
+        abortSignal?: AbortSignal;
+    }): Promise<{
+        tasks: ExecutionTaskApply[];
+    }>;
+    loadYamlFlowAsPlanning(userInstruction: string, yamlString: string): Promise<{
+        runner: TaskRunner;
+    }>;
+    runPlans(title: string, plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig): Promise<ExecutionResult>;
+    action(userPrompt: string, modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, includeBboxInPlanning: boolean, aiActContext?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, imagesIncludeCount?: number, deepThink?: DeepThinkOption, fileChooserAccept?: string[], deepLocate?: boolean, abortSignal?: AbortSignal): Promise<ExecutionResult<{
+        yamlFlow?: MidsceneYamlFlowItem[];
+        output?: string;
+    } | undefined>>;
+    private runAction;
+    private createTypeQueryTask;
+    createTypeQueryExecution<T>(type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert', demand: ServiceExtractParam, modelConfig: IModelConfig, opt?: ServiceExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<ExecutionResult<T>>;
+    waitFor(assertion: TUserPrompt, opt: PlanningActionParamWaitFor, modelConfig: IModelConfig): Promise<ExecutionResult<void>>;
+}
+export declare function withFileChooser<T>(interfaceInstance: AbstractInterface, fileChooserAccept: string[] | undefined, action: () => Promise<T>): Promise<T>;

package/dist/types/agent/ui-utils.d.ts

@@ -0,0 +1,14 @@
+import type { DetailedLocateParam, ExecutionTask, PullParam, ScrollParam } from '../types';
+export declare function typeStr(task: ExecutionTask): any;
+export declare function locateParamStr(locate?: DetailedLocateParam | string): string;
+export declare function scrollParamStr(scrollParam?: ScrollParam): string;
+export declare function pullParamStr(pullParam?: PullParam): string;
+export declare function extractInsightParam(taskParam: any): {
+    content: string;
+    images?: Array<{
+        name: string;
+        url: string;
+    }>;
+};
+export declare function taskTitleStr(type: 'Tap' | 'Hover' | 'Input' | 'RightClick' | 'KeyboardPress' | 'Scroll' | 'Act' | 'Query' | 'Assert' | 'WaitFor' | 'Locate' | 'Boolean' | 'Number' | 'String', prompt: string): string;
+export declare function paramStr(task: ExecutionTask): string;