@midscene/core 1.0.1-beta-20251022061922.0 → 1.0.1-beta-20251024064637.0

package/dist/lib/yaml/utils.js

@@ -44,11 +44,17 @@ const external_js_yaml_namespaceObject = require("js-yaml");
 var external_js_yaml_default = /*#__PURE__*/ __webpack_require__.n(external_js_yaml_namespaceObject);
 const debugUtils = (0, logger_namespaceObject.getDebug)('yaml:utils');
 function interpolateEnvVars(content) {
-    return content.replace(/\$\{([^}]+)\}/g, (_, envVar)=>{
-        const value = process.env[envVar.trim()];
-        if (void 0 === value) throw new Error(`Environment variable "${envVar.trim()}" is not defined`);
-        return value;
+    const lines = content.split('\n');
+    const processedLines = lines.map((line)=>{
+        const trimmedLine = line.trimStart();
+        if (trimmedLine.startsWith('#')) return line;
+        return line.replace(/\$\{([^}]+)\}/g, (_, envVar)=>{
+            const value = process.env[envVar.trim()];
+            if (void 0 === value) throw new Error(`Environment variable "${envVar.trim()}" is not defined`);
+            return value;
+        });
     });
+    return processedLines.join('\n');
 }
 function parseYamlScript(content, filePath) {
     let processedContent = content;

package/dist/lib/yaml/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"yaml/utils.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/yaml/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TUserPrompt } from '@/ai-model/common';\nimport type {\n  DetailedLocateParam,\n  LocateOption,\n  MidsceneYamlScript,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\n\nconst debugUtils = getDebug('yaml:utils');\n\nexport function interpolateEnvVars(content: string): string {\n  return content.replace(/\\$\\{([^}]+)\\}/g, (_, envVar) => {\n    const value = process.env[envVar.trim()];\n    if (value === undefined) {\n      throw new Error(`Environment variable \"${envVar.trim()}\" is not defined`);\n    }\n    return value;\n  });\n}\n\nexport function parseYamlScript(\n  content: string,\n  filePath?: string,\n): MidsceneYamlScript {\n  let processedContent = content;\n  if (content.indexOf('android') !== -1 && content.match(/deviceId:\\s*(\\d+)/)) {\n    let matchedDeviceId;\n    processedContent = content.replace(\n      /deviceId:\\s*(\\d+)/g,\n      (match, deviceId) => {\n        matchedDeviceId = deviceId;\n        return `deviceId: '${deviceId}'`;\n      },\n    );\n    console.warn(\n      `please use string-style deviceId in yaml script, for example: deviceId: \"${matchedDeviceId}\"`,\n    );\n  }\n  const interpolatedContent = interpolateEnvVars(processedContent);\n  const obj = yaml.load(interpolatedContent, {\n    schema: yaml.JSON_SCHEMA,\n  }) as MidsceneYamlScript;\n\n  const pathTip = filePath ? `, failed to load ${filePath}` : '';\n  assert(obj.tasks, `property \"tasks\" is required in yaml script ${pathTip}`);\n  assert(\n    Array.isArray(obj.tasks),\n    `property \"tasks\" must be an array in yaml script, but got ${obj.tasks}`,\n  );\n  return obj;\n}\n\nexport function buildDetailedLocateParam(\n  locatePrompt: TUserPrompt,\n  opt?: LocateOption,\n): DetailedLocateParam | undefined {\n  debugUtils('will call buildDetailedLocateParam', locatePrompt, opt);\n  let prompt = locatePrompt || opt?.prompt || (opt as any)?.locate; // as a shortcut\n  let deepThink = false;\n  let cacheable = true;\n  let xpath = undefined;\n\n  if (typeof opt === 'object' && opt !== null) {\n    deepThink = opt.deepThink ?? false;\n    cacheable = opt.cacheable ?? true;\n    xpath = opt.xpath;\n    if (locatePrompt && opt.prompt && locatePrompt !== opt.prompt) {\n      console.warn(\n        'conflict prompt for item',\n        locatePrompt,\n        opt,\n        'maybe you put the prompt in the wrong place',\n      );\n    }\n    prompt = prompt || opt.prompt;\n  }\n\n  if (!prompt) {\n    debugUtils(\n      'no prompt, will return undefined in buildDetailedLocateParam',\n      opt,\n    );\n    return undefined;\n  }\n\n  return {\n    prompt,\n    deepThink,\n    cacheable,\n    xpath,\n  };\n}\n\nexport function buildDetailedLocateParamAndRestParams(\n  locatePrompt: TUserPrompt,\n  opt: LocateOption | undefined,\n  excludeKeys: string[] = [],\n): {\n  locateParam: DetailedLocateParam | undefined;\n  restParams: Record<string, any>;\n} {\n  const locateParam = buildDetailedLocateParam(locatePrompt, opt);\n\n  // Extract all keys from opt except the ones already included in locateParam\n  const restParams: Record<string, any> = {};\n\n  if (typeof opt === 'object' && opt !== null) {\n    // Get all keys from opt\n    const allKeys = Object.keys(opt);\n\n    // Keys already included in locateParam: prompt, deepThink, cacheable, xpath\n    const locateParamKeys = Object.keys(locateParam || {});\n\n    // Extract all other keys\n    for (const key of allKeys) {\n      if (\n        !locateParamKeys.includes(key) &&\n        !excludeKeys.includes(key) &&\n        key !== 'locate'\n      ) {\n        restParams[key] = opt[key as keyof LocateOption];\n      }\n    }\n  }\n\n  return {\n    locateParam,\n    restParams,\n  };\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","debugUtils","getDebug","interpolateEnvVars","content","_","envVar","value","process","undefined","Error","parseYamlScript","filePath","processedContent","matchedDeviceId","match","deviceId","console","interpolatedContent","yaml","pathTip","assert","Array","buildDetailedLocateParam","locatePrompt","opt","prompt","deepThink","cacheable","xpath","buildDetailedLocateParamAndRestParams","excludeKeys","locateParam","restParams","allKeys","locateParamKeys"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACIA,MAAMI,aAAaC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAErB,SAASC,mBAAmBC,OAAe;IAChD,OAAOA,QAAQ,OAAO,CAAC,kBAAkB,CAACC,GAAGC;QAC3C,MAAMC,QAAQC,QAAQ,GAAG,CAACF,OAAO,IAAI,GAAG;QACxC,IAAIC,AAAUE,WAAVF,OACF,MAAM,IAAIG,MAAM,CAAC,sBAAsB,EAAEJ,OAAO,IAAI,GAAG,gBAAgB,CAAC;QAE1E,OAAOC;IACT;AACF;AAEO,SAASI,gBACdP,OAAe,EACfQ,QAAiB;IAEjB,IAAIC,mBAAmBT;IACvB,IAAIA,AAA+B,OAA/BA,QAAQ,OAAO,CAAC,cAAqBA,QAAQ,KAAK,CAAC,sBAAsB;QAC3E,IAAIU;QACJD,mBAAmBT,QAAQ,OAAO,CAChC,sBACA,CAACW,OAAOC;YACNF,kBAAkBE;YAClB,OAAO,CAAC,WAAW,EAAEA,SAAS,CAAC,CAAC;QAClC;QAEFC,QAAQ,IAAI,CACV,CAAC,yEAAyE,EAAEH,gBAAgB,CAAC,CAAC;IAElG;IACA,MAAMI,sBAAsBf,mBAAmBU;IAC/C,MAAMf,MAAMqB,2BAAAA,IAAS,CAACD,qBAAqB;QACzC,QAAQC,AAAAA,2BAAAA,WAAgB;IAC1B;IAEA,MAAMC,UAAUR,WAAW,CAAC,iBAAiB,EAAEA,UAAU,GAAG;IAC5DS,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOvB,IAAI,KAAK,EAAE,CAAC,4CAA4C,EAAEsB,SAAS;IAC1EC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEC,MAAM,OAAO,CAACxB,IAAI,KAAK,GACvB,CAAC,0DAA0D,EAAEA,IAAI,KAAK,EAAE;IAE1E,OAAOA;AACT;AAEO,SAASyB,yBACdC,YAAyB,EACzBC,GAAkB;IAElBxB,WAAW,sCAAsCuB,cAAcC;IAC/D,IAAIC,SAASF,gBAAgBC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,MAAM,AAAD,KAAMA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAa,MAAM,AAAD;IAC/D,IAAIE,YAAY;IAChB,IAAIC,YAAY;IAChB,IAAIC;IAEJ,IAAI,AAAe,YAAf,OAAOJ,OAAoBA,AAAQ,SAARA,KAAc;QAC3CE,YAAYF,IAAI,SAAS,IAAI;QAC7BG,YAAYH,IAAI,SAAS,IAAI;QAC7BI,QAAQJ,IAAI,KAAK;QACjB,IAAID,gBAAgBC,IAAI,MAAM,IAAID,iBAAiBC,IAAI,MAAM,EAC3DR,QAAQ,IAAI,CACV,4BACAO,cACAC,KACA;QAGJC,SAASA,UAAUD,IAAI,MAAM;IAC/B;IAEA,IAAI,CAACC,QAAQ,YACXzB,WACE,gEACAwB;IAKJ,OAAO;QACLC;QACAC;QACAC;QACAC;IACF;AACF;AAEO,SAASC,sCACdN,YAAyB,EACzBC,GAA6B,EAC7BM,cAAwB,EAAE;IAK1B,MAAMC,cAAcT,yBAAyBC,cAAcC;IAG3D,MAAMQ,aAAkC,CAAC;IAEzC,IAAI,AAAe,YAAf,OAAOR,OAAoBA,AAAQ,SAARA,KAAc;QAE3C,MAAMS,UAAUrC,OAAO,IAAI,CAAC4B;QAG5B,MAAMU,kBAAkBtC,OAAO,IAAI,CAACmC,eAAe,CAAC;QAGpD,KAAK,MAAMpC,OAAOsC,QAChB,IACE,CAACC,gBAAgB,QAAQ,CAACvC,QAC1B,CAACmC,YAAY,QAAQ,CAACnC,QACtBA,AAAQ,aAARA,KAEAqC,UAAU,CAACrC,IAAI,GAAG6B,GAAG,CAAC7B,IAA0B;IAGtD;IAEA,OAAO;QACLoC;QACAC;IACF;AACF"}
+{"version":3,"file":"yaml/utils.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/yaml/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TUserPrompt } from '@/ai-model/common';\nimport type {\n  DetailedLocateParam,\n  LocateOption,\n  MidsceneYamlScript,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\n\nconst debugUtils = getDebug('yaml:utils');\n\nexport function interpolateEnvVars(content: string): string {\n  // Process line by line to skip commented lines\n  const lines = content.split('\\n');\n  const processedLines = lines.map((line) => {\n    // Check if the line is a YAML comment (starts with # after optional whitespace)\n    const trimmedLine = line.trimStart();\n    if (trimmedLine.startsWith('#')) {\n      // Skip interpolation for comment lines\n      return line;\n    }\n\n    // Process environment variables for non-comment lines\n    return line.replace(/\\$\\{([^}]+)\\}/g, (_, envVar) => {\n      const value = process.env[envVar.trim()];\n      if (value === undefined) {\n        throw new Error(\n          `Environment variable \"${envVar.trim()}\" is not defined`,\n        );\n      }\n      return value;\n    });\n  });\n\n  return processedLines.join('\\n');\n}\n\nexport function parseYamlScript(\n  content: string,\n  filePath?: string,\n): MidsceneYamlScript {\n  let processedContent = content;\n  if (content.indexOf('android') !== -1 && content.match(/deviceId:\\s*(\\d+)/)) {\n    let matchedDeviceId;\n    processedContent = content.replace(\n      /deviceId:\\s*(\\d+)/g,\n      (match, deviceId) => {\n        matchedDeviceId = deviceId;\n        return `deviceId: '${deviceId}'`;\n      },\n    );\n    console.warn(\n      `please use string-style deviceId in yaml script, for example: deviceId: \"${matchedDeviceId}\"`,\n    );\n  }\n  const interpolatedContent = interpolateEnvVars(processedContent);\n  const obj = yaml.load(interpolatedContent, {\n    schema: yaml.JSON_SCHEMA,\n  }) as MidsceneYamlScript;\n\n  const pathTip = filePath ? `, failed to load ${filePath}` : '';\n  assert(obj.tasks, `property \"tasks\" is required in yaml script ${pathTip}`);\n  assert(\n    Array.isArray(obj.tasks),\n    `property \"tasks\" must be an array in yaml script, but got ${obj.tasks}`,\n  );\n  return obj;\n}\n\nexport function buildDetailedLocateParam(\n  locatePrompt: TUserPrompt,\n  opt?: LocateOption,\n): DetailedLocateParam | undefined {\n  debugUtils('will call buildDetailedLocateParam', locatePrompt, opt);\n  let prompt = locatePrompt || opt?.prompt || (opt as any)?.locate; // as a shortcut\n  let deepThink = false;\n  let cacheable = true;\n  let xpath = undefined;\n\n  if (typeof opt === 'object' && opt !== null) {\n    deepThink = opt.deepThink ?? false;\n    cacheable = opt.cacheable ?? true;\n    xpath = opt.xpath;\n    if (locatePrompt && opt.prompt && locatePrompt !== opt.prompt) {\n      console.warn(\n        'conflict prompt for item',\n        locatePrompt,\n        opt,\n        'maybe you put the prompt in the wrong place',\n      );\n    }\n    prompt = prompt || opt.prompt;\n  }\n\n  if (!prompt) {\n    debugUtils(\n      'no prompt, will return undefined in buildDetailedLocateParam',\n      opt,\n    );\n    return undefined;\n  }\n\n  return {\n    prompt,\n    deepThink,\n    cacheable,\n    xpath,\n  };\n}\n\nexport function buildDetailedLocateParamAndRestParams(\n  locatePrompt: TUserPrompt,\n  opt: LocateOption | undefined,\n  excludeKeys: string[] = [],\n): {\n  locateParam: DetailedLocateParam | undefined;\n  restParams: Record<string, any>;\n} {\n  const locateParam = buildDetailedLocateParam(locatePrompt, opt);\n\n  // Extract all keys from opt except the ones already included in locateParam\n  const restParams: Record<string, any> = {};\n\n  if (typeof opt === 'object' && opt !== null) {\n    // Get all keys from opt\n    const allKeys = Object.keys(opt);\n\n    // Keys already included in locateParam: prompt, deepThink, cacheable, xpath\n    const locateParamKeys = Object.keys(locateParam || {});\n\n    // Extract all other keys\n    for (const key of allKeys) {\n      if (\n        !locateParamKeys.includes(key) &&\n        !excludeKeys.includes(key) &&\n        key !== 'locate'\n      ) {\n        restParams[key] = opt[key as keyof LocateOption];\n      }\n    }\n  }\n\n  return {\n    locateParam,\n    restParams,\n  };\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","debugUtils","getDebug","interpolateEnvVars","content","lines","processedLines","line","trimmedLine","_","envVar","value","process","undefined","Error","parseYamlScript","filePath","processedContent","matchedDeviceId","match","deviceId","console","interpolatedContent","yaml","pathTip","assert","Array","buildDetailedLocateParam","locatePrompt","opt","prompt","deepThink","cacheable","xpath","buildDetailedLocateParamAndRestParams","excludeKeys","locateParam","restParams","allKeys","locateParamKeys"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACIA,MAAMI,aAAaC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAErB,SAASC,mBAAmBC,OAAe;IAEhD,MAAMC,QAAQD,QAAQ,KAAK,CAAC;IAC5B,MAAME,iBAAiBD,MAAM,GAAG,CAAC,CAACE;QAEhC,MAAMC,cAAcD,KAAK,SAAS;QAClC,IAAIC,YAAY,UAAU,CAAC,MAEzB,OAAOD;QAIT,OAAOA,KAAK,OAAO,CAAC,kBAAkB,CAACE,GAAGC;YACxC,MAAMC,QAAQC,QAAQ,GAAG,CAACF,OAAO,IAAI,GAAG;YACxC,IAAIC,AAAUE,WAAVF,OACF,MAAM,IAAIG,MACR,CAAC,sBAAsB,EAAEJ,OAAO,IAAI,GAAG,gBAAgB,CAAC;YAG5D,OAAOC;QACT;IACF;IAEA,OAAOL,eAAe,IAAI,CAAC;AAC7B;AAEO,SAASS,gBACdX,OAAe,EACfY,QAAiB;IAEjB,IAAIC,mBAAmBb;IACvB,IAAIA,AAA+B,OAA/BA,QAAQ,OAAO,CAAC,cAAqBA,QAAQ,KAAK,CAAC,sBAAsB;QAC3E,IAAIc;QACJD,mBAAmBb,QAAQ,OAAO,CAChC,sBACA,CAACe,OAAOC;YACNF,kBAAkBE;YAClB,OAAO,CAAC,WAAW,EAAEA,SAAS,CAAC,CAAC;QAClC;QAEFC,QAAQ,IAAI,CACV,CAAC,yEAAyE,EAAEH,gBAAgB,CAAC,CAAC;IAElG;IACA,MAAMI,sBAAsBnB,mBAAmBc;IAC/C,MAAMnB,MAAMyB,2BAAAA,IAAS,CAACD,qBAAqB;QACzC,QAAQC,AAAAA,2BAAAA,WAAgB;IAC1B;IAEA,MAAMC,UAAUR,WAAW,CAAC,iBAAiB,EAAEA,UAAU,GAAG;IAC5DS,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO3B,IAAI,KAAK,EAAE,CAAC,4CAA4C,EAAE0B,SAAS;IAC1EC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEC,MAAM,OAAO,CAAC5B,IAAI,KAAK,GACvB,CAAC,0DAA0D,EAAEA,IAAI,KAAK,EAAE;IAE1E,OAAOA;AACT;AAEO,SAAS6B,yBACdC,YAAyB,EACzBC,GAAkB;IAElB5B,WAAW,sCAAsC2B,cAAcC;IAC/D,IAAIC,SAASF,gBAAgBC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,MAAM,AAAD,KAAMA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAa,MAAM,AAAD;IAC/D,IAAIE,YAAY;IAChB,IAAIC,YAAY;IAChB,IAAIC;IAEJ,IAAI,AAAe,YAAf,OAAOJ,OAAoBA,AAAQ,SAARA,KAAc;QAC3CE,YAAYF,IAAI,SAAS,IAAI;QAC7BG,YAAYH,IAAI,SAAS,IAAI;QAC7BI,QAAQJ,IAAI,KAAK;QACjB,IAAID,gBAAgBC,IAAI,MAAM,IAAID,iBAAiBC,IAAI,MAAM,EAC3DR,QAAQ,IAAI,CACV,4BACAO,cACAC,KACA;QAGJC,SAASA,UAAUD,IAAI,MAAM;IAC/B;IAEA,IAAI,CAACC,QAAQ,YACX7B,WACE,gEACA4B;IAKJ,OAAO;QACLC;QACAC;QACAC;QACAC;IACF;AACF;AAEO,SAASC,sCACdN,YAAyB,EACzBC,GAA6B,EAC7BM,cAAwB,EAAE;IAK1B,MAAMC,cAAcT,yBAAyBC,cAAcC;IAG3D,MAAMQ,aAAkC,CAAC;IAEzC,IAAI,AAAe,YAAf,OAAOR,OAAoBA,AAAQ,SAARA,KAAc;QAE3C,MAAMS,UAAUzC,OAAO,IAAI,CAACgC;QAG5B,MAAMU,kBAAkB1C,OAAO,IAAI,CAACuC,eAAe,CAAC;QAGpD,KAAK,MAAMxC,OAAO0C,QAChB,IACE,CAACC,gBAAgB,QAAQ,CAAC3C,QAC1B,CAACuC,YAAY,QAAQ,CAACvC,QACtBA,AAAQ,aAARA,KAEAyC,UAAU,CAACzC,IAAI,GAAGiC,GAAG,CAACjC,IAA0B;IAGtD;IAEA,OAAO;QACLwC;QACAC;IACF;AACF"}

package/dist/types/agent/index.d.ts

@@ -5,5 +5,5 @@ export { locateParamStr, paramStr, taskTitleStr, typeStr } from './ui-utils';
 export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
 export { cacheFileExt } from './task-cache';
 export { TaskExecutor } from './tasks';
-export { getCurrentExecutionFile, trimContextByViewport, } from './utils';
+export { getCurrentExecutionFile } from './utils';
 export type { AgentOpt } from '../types';

package/dist/types/agent/utils.d.ts

@@ -1,6 +1,6 @@
 import type { TMultimodalPrompt, TUserPrompt } from '../ai-model/common';
 import type { AbstractInterface } from '../device';
-import type { BaseElement, ElementCacheFeature, ElementTreeNode, ExecutionDump, ExecutorContext, LocateResultElement, PlanningLocateParam, UIContext } from '../types';
+import type { ElementCacheFeature, LocateResultElement, PlanningLocateParam, UIContext } from '../types';
 import type { TaskCache } from './task-cache';
 export declare function commonContextParser(interfaceInstance: AbstractInterface, _opt: {
     uploadServerUrl?: string;
@@ -13,42 +13,11 @@ export declare function printReportMsg(filepath: string): void;
  */
 export declare function getCurrentExecutionFile(trace?: string): string | false;
 export declare function generateCacheId(fileName?: string): string;
-export declare function matchElementFromPlan(planLocateParam: PlanningLocateParam, tree: ElementTreeNode<BaseElement>): any;
+export declare function matchElementFromPlan(planLocateParam: PlanningLocateParam): LocateResultElement | undefined;
 export declare function matchElementFromCache(context: {
     taskCache?: TaskCache;
     interfaceInstance: AbstractInterface;
 }, cacheEntry: ElementCacheFeature | undefined, cachePrompt: TUserPrompt, cacheable: boolean | undefined): Promise<LocateResultElement | undefined>;
-export declare function trimContextByViewport(execution: ExecutionDump): {
-    tasks: {
-        type: any;
-        subType?: string;
-        subTask?: boolean;
-        param?: any;
-        thought?: string;
-        locate?: PlanningLocateParam | null;
-        uiContext?: UIContext;
-        executor: (param: any, context: ExecutorContext) => void | Promise<void | import("../types").ExecutionTaskReturn<any, any> | undefined> | undefined;
-        output?: any;
-        log?: any;
-        recorder?: import("../types").ExecutionRecorderItem[];
-        hitBy?: import("../types").ExecutionTaskHitBy;
-        status: "pending" | "running" | "finished" | "failed" | "cancelled";
-        error?: Error;
-        errorMessage?: string;
-        errorStack?: string;
-        timing?: {
-            start: number;
-            end?: number;
-            cost?: number;
-        };
-        usage?: import("../types").AIUsageInfo;
-        searchAreaUsage?: import("../types").AIUsageInfo;
-    }[];
-    name: string;
-    description?: string;
-    aiActionContext?: string;
-    logTime: number;
-};
 export declare const getMidsceneVersion: () => string;
 export declare const parsePrompt: (prompt: TUserPrompt) => {
     textPrompt: string;

package/dist/types/ai-model/index.d.ts

@@ -1,6 +1,6 @@
 export { callAIWithStringResponse, callAIWithObjectResponse, callAI, } from './service-caller/index';
 export { systemPromptToLocateElement } from './prompt/llm-locator';
-export { describeUserPage, elementByPositionWithElementInfo, } from './prompt/util';
+export { describeUserPage } from './prompt/util';
 export { generatePlaywrightTest, generatePlaywrightTestStream, } from './prompt/playwright-generator';
 export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generator';
 export type { ChatCompletionMessageParam } from 'openai/resources/index';

package/dist/types/ai-model/inspect.d.ts

@@ -1,5 +1,6 @@
-import type { AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, UIContext } from '../types';
+import type { AIDataExtractionResponse, AIElementResponse, AIUsageInfo, InsightExtractOption, Rect, ReferenceImage, UIContext } from '../types';
 import type { IModelConfig } from '@midscene/shared/env';
+import type { LocateResultElement } from '@midscene/shared/types';
 import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
 import type { TMultimodalPrompt, TUserPrompt } from './common';
 import { callAIWithObjectResponse } from './service-caller/index';
@@ -7,23 +8,24 @@ export type AIArgs = [
     ChatCompletionSystemMessageParam,
     ...ChatCompletionUserMessageParam[]
 ];
-export declare function AiLocateElement<ElementType extends BaseElement = BaseElement>(options: {
-    context: UIContext<ElementType>;
+export declare function AiLocateElement(options: {
+    context: UIContext;
     targetElementDescription: TUserPrompt;
     referenceImage?: ReferenceImage;
     callAIFn: typeof callAIWithObjectResponse<AIElementResponse | [number, number]>;
     searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;
     modelConfig: IModelConfig;
 }): Promise<{
-    parseResult: AIElementLocatorResponse;
+    parseResult: {
+        elements: LocateResultElement[];
+        errors?: string[];
+    };
     rect?: Rect;
     rawResponse: string;
-    elementById: ElementById;
     usage?: AIUsageInfo;
-    isOrderSensitive?: boolean;
 }>;
 export declare function AiLocateSection(options: {
-    context: UIContext<BaseElement>;
+    context: UIContext;
     sectionDescription: TUserPrompt;
     modelConfig: IModelConfig;
 }): Promise<{
@@ -33,14 +35,14 @@ export declare function AiLocateSection(options: {
     rawResponse: string;
     usage?: AIUsageInfo;
 }>;
-export declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseElement>(options: {
+export declare function AiExtractElementInfo<T>(options: {
     dataQuery: string | Record<string, string>;
     multimodalPrompt?: TMultimodalPrompt;
-    context: UIContext<ElementType>;
+    context: UIContext;
+    pageDescription?: string;
     extractOption?: InsightExtractOption;
     modelConfig: IModelConfig;
 }): Promise<{
     parseResult: AIDataExtractionResponse<T>;
-    elementById: (idOrIndexId: string) => ElementType;
     usage: AIUsageInfo | undefined;
 }>;

package/dist/types/ai-model/prompt/util.d.ts

@@ -1,17 +1,9 @@
-import type { BaseElement, ElementTreeNode, Size, UIContext } from '../../types';
-import type { TVlModeTypes } from '@midscene/shared/env';
+import type { BaseElement, Size, UIContext } from '../../types';
 export declare function describeSize(size: Size): string;
 export declare function describeElement(elements: (Pick<BaseElement, 'rect' | 'content'> & {
     id: string;
 })[]): string;
 export declare const distanceThreshold = 16;
-export declare function elementByPositionWithElementInfo(treeRoot: ElementTreeNode<BaseElement>, position: {
-    x: number;
-    y: number;
-}, options?: {
-    requireStrictDistance?: boolean;
-    filterPositionElements?: boolean;
-}): BaseElement | undefined;
 export declare function distance(point1: {
     x: number;
     y: number;
@@ -20,28 +12,4 @@ export declare function distance(point1: {
     y: number;
 }): number;
 export declare const samplePageDescription = "\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n";
-export declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>, opt: {
-    truncateTextLength?: number;
-    filterNonTextContent?: boolean;
-    domIncluded?: boolean | 'visible-only';
-    visibleOnly?: boolean;
-    vlMode: TVlModeTypes | undefined;
-}): Promise<{
-    description: string;
-    elementById(idOrIndexId: string): ElementType;
-    elementByPosition(position: {
-        x: number;
-        y: number;
-    }, size: {
-        width: number;
-        height: number;
-    }): BaseElement | undefined;
-    insertElementByPosition(position: {
-        x: number;
-        y: number;
-    }): ElementType;
-    size: {
-        width: number;
-        height: number;
-    };
-}>;
+export declare function describeUserPage(context: UIContext): Promise<string>;

package/dist/types/insight/index.d.ts

@@ -1,9 +1,9 @@
 import { callAIWithObjectResponse } from '../ai-model/index';
-import type { AIDescribeElementResponse, BaseElement, DetailedLocateParam, InsightExtractOption, InsightExtractParam, InsightExtractResult, InsightTaskInfo, LocateResultWithDump, Rect, UIContext } from '../types';
+import type { AIDescribeElementResponse, DetailedLocateParam, InsightExtractOption, InsightExtractParam, InsightExtractResult, InsightTaskInfo, LocateResultWithDump, Rect, UIContext } from '../types';
 import { type IModelConfig } from '@midscene/shared/env';
 import type { TMultimodalPrompt } from '../ai-model/common';
 export interface LocateOpts {
-    context?: UIContext<BaseElement>;
+    context?: UIContext;
 }
 export type AnyValue<T> = {
     [K in keyof T]: unknown extends T[K] ? any : T[K];
@@ -12,13 +12,13 @@ interface InsightOptions {
     taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
     aiVendorFn?: typeof callAIWithObjectResponse;
 }
-export default class Insight<ElementType extends BaseElement = BaseElement, ContextType extends UIContext<ElementType> = UIContext<ElementType>> {
-    contextRetrieverFn: () => Promise<ContextType> | ContextType;
+export default class Insight {
+    contextRetrieverFn: () => Promise<UIContext> | UIContext;
     aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined>;
     taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
-    constructor(context: ContextType | (() => Promise<ContextType> | ContextType), opt?: InsightOptions);
+    constructor(context: UIContext | (() => Promise<UIContext> | UIContext), opt?: InsightOptions);
     locate(query: DetailedLocateParam, opt: LocateOpts, modelConfig: IModelConfig): Promise<LocateResultWithDump>;
-    extract<T>(dataDemand: InsightExtractParam, modelConfig: IModelConfig, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<InsightExtractResult<T>>;
+    extract<T>(dataDemand: InsightExtractParam, modelConfig: IModelConfig, opt?: InsightExtractOption, pageDescription?: string, multimodalPrompt?: TMultimodalPrompt): Promise<InsightExtractResult<T>>;
     describe(target: Rect | [number, number], modelConfig: IModelConfig, opt?: {
         deepThink?: boolean;
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;

package/dist/types/tree.d.ts

@@ -1 +1,4 @@
-export { truncateText, trimAttributes, descriptionOfTree, } from '@midscene/shared/extractor';
+import type { BaseElement, ElementTreeNode } from '@midscene/shared/types';
+import { trimAttributes, truncateText } from '@midscene/shared/extractor';
+export { trimAttributes, truncateText };
+export declare function descriptionOfTree<ElementType extends BaseElement = BaseElement>(tree: ElementTreeNode<ElementType>, truncateTextLength?: number, filterNonTextContent?: boolean, visibleOnly?: boolean): string;

package/dist/types/types.d.ts

@@ -1,6 +1,6 @@
 import type { NodeType } from '@midscene/shared/constants';
 import type { CreateOpenAIClientFn, TModelConfigFn } from '@midscene/shared/env';
-import type { BaseElement, ElementTreeNode, Rect, Size } from '@midscene/shared/types';
+import type { BaseElement, LocateResultElement, Rect, Size } from '@midscene/shared/types';
 import type { z } from 'zod';
 import type { TUserPrompt } from './ai-model/common';
 import type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';
@@ -15,6 +15,7 @@ export type AIUsageInfo = Record<string, any> & {
     model_description: string | undefined;
     intent: string | undefined;
 };
+export type { LocateResultElement };
 /**
  * openai
  *
@@ -39,23 +40,12 @@ export type AISingleElementResponseByPosition = {
     text: string;
 };
 export type AISingleElementResponse = AISingleElementResponseById;
-export interface AIElementLocatorResponse {
-    elements: {
-        id: string;
-        reason?: string;
-        text?: string;
-        xpaths?: string[];
-    }[];
-    bbox?: [number, number, number, number];
-    isOrderSensitive?: boolean;
-    errors?: string[];
-}
 export interface AIElementCoordinatesResponse {
     bbox: [number, number, number, number];
     isOrderSensitive?: boolean;
     errors?: string[];
 }
-export type AIElementResponse = AIElementLocatorResponse | AIElementCoordinatesResponse;
+export type AIElementResponse = AIElementCoordinatesResponse;
 export interface AIDataExtractionResponse<DataDemand> {
     data: DataDemand;
     errors?: string[];
@@ -91,9 +81,8 @@ export interface AgentDescribeElementAtPointResult {
 /**
  * context
  */
-export declare abstract class UIContext<ElementType extends BaseElement = BaseElement> {
+export declare abstract class UIContext {
     abstract screenshotBase64: string;
-    abstract tree: ElementTreeNode<ElementType>;
     abstract size: Size;
     abstract _isFrozen?: boolean;
 }
@@ -103,18 +92,6 @@ export type EnsureObject<T> = {
 export type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';
 export type InsightExtractParam = string | Record<string, string>;
 export type ElementCacheFeature = Record<string, unknown>;
-export type LocateResultElement = {
-    center: [number, number];
-    rect: Rect;
-    id: string;
-    indexId?: number;
-    xpaths: string[];
-    attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    isOrderSensitive?: boolean;
-};
 export interface LocateResult {
     element: LocateResultElement | null;
     rect?: Rect;
@@ -143,7 +120,7 @@ export interface InsightDump extends DumpMeta {
         dataDemand?: InsightExtractParam;
         assertion?: TUserPrompt;
     };
-    matchedElement: BaseElement[];
+    matchedElement: LocateResultElement[];
     matchedRect?: Rect;
     deepThink?: boolean;
     data: any;
@@ -193,7 +170,6 @@ export interface AgentAssertOpt {
  *
  */
 export interface PlanningLocateParam extends DetailedLocateParam {
-    id?: string;
     bbox?: [number, number, number, number];
 }
 export interface PlanningAction<ParamType = any> {
@@ -385,7 +361,7 @@ export interface WebElementInfo extends BaseElement {
         [key: string]: string;
     };
 }
-export type WebUIContext = UIContext<WebElementInfo>;
+export type WebUIContext = UIContext;
 /**
  * Agent
  */

package/dist/types/yaml.d.ts

@@ -1,12 +1,12 @@
 import type { TUserPrompt } from './ai-model/common';
 import type { AgentOpt, Rect } from './types';
-import type { BaseElement, UIContext } from './types';
+import type { UIContext } from './types';
 export interface LocateOption {
     prompt?: TUserPrompt;
     deepThink?: boolean;
     cacheable?: boolean;
     xpath?: string;
-    uiContext?: UIContext<BaseElement>;
+    uiContext?: UIContext;
 }
 export interface InsightExtractOption {
     domIncluded?: boolean | 'visible-only';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.0.1-beta-20251022061922.0",
+  "version": "1.0.1-beta-20251024064637.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -89,8 +89,8 @@
     "zod": "3.24.3",
     "semver": "7.5.2",
     "js-yaml": "4.1.0",
-    "@midscene/recorder": "1.0.1-beta-20251022061922.0",
-    "@midscene/shared": "1.0.1-beta-20251022061922.0"
+    "@midscene/recorder": "1.0.1-beta-20251024064637.0",
+    "@midscene/shared": "1.0.1-beta-20251024064637.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.11.2",