@midscene/core 1.10.8-beta-20260723115348.0 → 1.10.8

package/dist/es/yaml/utils.mjs

@@ -1,6 +1,7 @@
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import js_yaml from "js-yaml";
+import { buildLocatePromptWithContext } from "../agent/prompt-context.mjs";
 const debugUtils = getDebug('yaml:utils');
 const topLevelTasksPattern = /^tasks\s*:/;
 const topLevelYamlKeyPattern = /^[^\s#][^:]*:/;
@@ -140,6 +141,9 @@ function buildDetailedLocateParam(locatePrompt, opt) {
         prompt = prompt || opt.prompt;
     }
     if (!prompt) return void debugUtils('no prompt, will return undefined in buildDetailedLocateParam', opt);
+    const promptDisplay = 'string' == typeof prompt ? prompt : prompt.prompt;
+    const context = opt?.context?.trim() || void 0;
+    prompt = buildLocatePromptWithContext(prompt, opt?.context);
     const multimodalPrompt = extractMultimodalPrompt(opt);
     if (multimodalPrompt) prompt = 'string' == typeof prompt ? {
         prompt,
@@ -150,6 +154,10 @@ function buildDetailedLocateParam(locatePrompt, opt) {
     };
     return {
         prompt,
+        ...context ? {
+            promptDisplay,
+            context
+        } : {},
         deepLocate,
         cacheable,
         xpath
@@ -163,7 +171,7 @@ function buildDetailedLocateParamAndRestParams(locatePrompt, opt, excludeKeys =
         const allKeys = Object.keys(opt);
         const locateParamKeys = Object.keys(locateParam || {});
         const multimodalPromptKeys = 'object' == typeof locateParam?.prompt && locateParam?.prompt !== null ? Object.keys(multimodalPrompt || {}) : [];
-        for (const key of allKeys)if (!locateParamKeys.includes(key) && !multimodalPromptKeys.includes(key) && !excludeKeys.includes(key) && 'locate' !== key) restParams[key] = opt[key];
+        for (const key of allKeys)if (!locateParamKeys.includes(key) && 'context' !== key && 'deepThink' !== key && !multimodalPromptKeys.includes(key) && !excludeKeys.includes(key) && 'locate' !== key) restParams[key] = opt[key];
     }
     return {
         locateParam,

package/dist/es/yaml/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"yaml/utils.mjs","sources":["../../../src/yaml/utils.ts"],"sourcesContent":["import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type {\n  DetailedLocateParam,\n  LocateOption,\n  MidsceneYamlScript,\n  MidsceneYamlScriptWebEnv,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\n\nconst debugUtils = getDebug('yaml:utils');\n\nconst topLevelTasksPattern = /^tasks\\s*:/;\nconst topLevelYamlKeyPattern = /^[^\\s#][^:]*:/;\n\nexport type WebTargetSource = 'page' | 'browser' | 'web' | 'target';\n\nexport type ResolvedWebTarget = {\n  source: WebTargetSource;\n  target: Partial<MidsceneYamlScriptWebEnv> & { mode: 'page' | 'browser' };\n  mode: 'page' | 'browser';\n};\n\nexport type WebTargetConfig = Partial<\n  Record<WebTargetSource, Partial<MidsceneYamlScriptWebEnv>>\n>;\n\nconst webTargetSources: WebTargetSource[] = [\n  'page',\n  'browser',\n  'web',\n  'target',\n];\n\nexport function resolveWebTarget(\n  config: WebTargetConfig,\n): ResolvedWebTarget | undefined {\n  const entries = webTargetSources\n    .map((source) => [source, config[source]] as const)\n    .filter(\n      (\n        entry,\n      ): entry is readonly [\n        WebTargetSource,\n        Partial<MidsceneYamlScriptWebEnv>,\n      ] => typeof entry[1] !== 'undefined',\n    );\n\n  if (entries.length === 0) {\n    return undefined;\n  }\n\n  if (entries.length > 1) {\n    const specifiedTargets = entries.map(([source]) => source);\n    throw new Error(\n      `[midscene] Only one web target can be specified, but found multiple: ${specifiedTargets.join(\n        ', ',\n      )}. Please specify only one of: page, browser, web, or target.`,\n    );\n  }\n\n  const [source, target] = entries[0];\n  const explicitMode = target.mode;\n  if (\n    typeof explicitMode !== 'undefined' &&\n    explicitMode !== 'page' &&\n    explicitMode !== 'browser'\n  ) {\n    throw new Error(\n      `[midscene] web target mode must be either \"page\" or \"browser\", but got \"${explicitMode}\".`,\n    );\n  }\n\n  if (source === 'page' && explicitMode === 'browser') {\n    throw new Error(\n      '[midscene] page target cannot use mode: browser. Use browser: instead.',\n    );\n  }\n\n  if (source === 'browser' && explicitMode === 'page') {\n    throw new Error(\n      '[midscene] browser target cannot use mode: page. Use page: instead.',\n    );\n  }\n\n  const mode =\n    source === 'page'\n      ? 'page'\n      : source === 'browser'\n        ? 'browser'\n        : (explicitMode ?? 'page');\n\n  if (mode === 'page' && target.autoFollowNewPage) {\n    throw new Error(\n      '[midscene] autoFollowNewPage requires browser mode. Use browser: or web.mode: browser.',\n    );\n  }\n\n  if (\n    mode === 'browser' &&\n    typeof target.forceSameTabNavigation !== 'undefined'\n  ) {\n    throw new Error(\n      '[midscene] forceSameTabNavigation cannot be used in browser mode. Use page: or web.mode: page when same-tab navigation is required.',\n    );\n  }\n\n  return {\n    source,\n    mode,\n    target: {\n      ...target,\n      mode,\n    },\n  };\n}\n\nfunction interpolateEnvVarRefs(\n  value: string,\n  keepUnresolvedRefs = false,\n): string {\n  let result = '';\n  let lastIndex = 0;\n  let searchFrom = 0;\n\n  while (searchFrom < value.length) {\n    const start = value.indexOf('${', searchFrom);\n    if (start === -1) {\n      break;\n    }\n\n    const end = value.indexOf('}', start + 2);\n    if (end === -1) {\n      break;\n    }\n\n    const rawName = value.slice(start + 2, end);\n    if (!rawName) {\n      searchFrom = end + 1;\n      continue;\n    }\n\n    result += value.slice(lastIndex, start);\n    const envVar = rawName.trim();\n    const envValue = process.env[envVar];\n    if (envValue === undefined) {\n      if (keepUnresolvedRefs) {\n        result += value.slice(start, end + 1);\n      } else {\n        throw new Error(`Environment variable \"${envVar}\" is not defined`);\n      }\n    } else {\n      result += envValue;\n    }\n    lastIndex = end + 1;\n    searchFrom = end + 1;\n  }\n\n  return result + value.slice(lastIndex);\n}\n\nconst multimodalLocateOptionFieldMap: Record<keyof TMultimodalPrompt, true> = {\n  images: true,\n  convertHttpImage2Base64: true,\n};\n\nconst multimodalLocateOptionKeys = Object.keys(\n  multimodalLocateOptionFieldMap,\n) as Array<keyof TMultimodalPrompt>;\n\nfunction extractMultimodalPrompt(\n  opt?: LocateOption,\n): Partial<TMultimodalPrompt> | undefined {\n  if (typeof opt !== 'object' || opt === null) {\n    return undefined;\n  }\n\n  const entries = multimodalLocateOptionKeys\n    .map((key) => [key, opt[key]] as const)\n    .filter(([, value]) => value !== undefined);\n\n  return entries.length\n    ? (Object.fromEntries(entries) as Partial<TMultimodalPrompt>)\n    : undefined;\n}\n\nexport function interpolateEnvVars(content: string): string {\n  // Process line by line to skip commented lines\n  const lines = content.split('\\n');\n  const processedLines = lines.map((line) => {\n    // Check if the line is a YAML comment (starts with # after optional whitespace)\n    const trimmedLine = line.trimStart();\n    if (trimmedLine.startsWith('#')) {\n      // Skip interpolation for comment lines\n      return line;\n    }\n\n    // Process environment variables for non-comment lines\n    return interpolateEnvVarRefs(line);\n  });\n\n  return processedLines.join('\\n');\n}\n\nfunction interpolateYamlScriptEnvVars(content: string): string {\n  let inTopLevelTasksBlock = false;\n\n  return content\n    .split('\\n')\n    .map((line) => {\n      const trimmedLine = line.trimStart();\n      if (!trimmedLine || trimmedLine.startsWith('#')) {\n        return line;\n      }\n\n      const indentSize = line.length - trimmedLine.length;\n      if (indentSize === 0) {\n        if (topLevelTasksPattern.test(trimmedLine)) {\n          inTopLevelTasksBlock = true;\n          return line;\n        }\n\n        if (topLevelYamlKeyPattern.test(trimmedLine)) {\n          inTopLevelTasksBlock = false;\n        }\n      }\n\n      return interpolateEnvVarRefs(line, inTopLevelTasksBlock);\n    })\n    .join('\\n');\n}\n\nexport function parseYamlScript(\n  content: string,\n  filePath?: string,\n): MidsceneYamlScript {\n  let processedContent = content;\n  if (content.indexOf('android') !== -1 && content.match(/deviceId:\\s*(\\d+)/)) {\n    let matchedDeviceId;\n    processedContent = content.replace(\n      /deviceId:\\s*(\\d+)/g,\n      (match, deviceId) => {\n        matchedDeviceId = deviceId;\n        return `deviceId: '${deviceId}'`;\n      },\n    );\n    console.warn(\n      `please use string-style deviceId in yaml script, for example: deviceId: \"${matchedDeviceId}\"`,\n    );\n  }\n  const interpolatedContent = interpolateYamlScriptEnvVars(processedContent);\n  const obj = yaml.load(interpolatedContent, {\n    schema: yaml.JSON_SCHEMA,\n  }) as MidsceneYamlScript;\n\n  const pathTip = filePath ? `, failed to load ${filePath}` : '';\n  resolveWebTarget(obj);\n  assert(obj.tasks, `property \"tasks\" is required in yaml script ${pathTip}`);\n  assert(\n    Array.isArray(obj.tasks),\n    `property \"tasks\" must be an array in yaml script, but got ${obj.tasks}`,\n  );\n  return obj;\n}\n\nexport function buildDetailedLocateParam(\n  locatePrompt: TUserPrompt,\n  opt?: LocateOption,\n): DetailedLocateParam | undefined {\n  debugUtils('will call buildDetailedLocateParam', locatePrompt, opt);\n  // Normalize object-form TUserPrompt: when the object only contains a\n  // `prompt` string (no multimodal fields like `images`), unwrap it to\n  // avoid double nesting like { prompt: { prompt: '...' } }.\n  let normalizedLocatePrompt: TUserPrompt = locatePrompt;\n  if (\n    typeof locatePrompt === 'object' &&\n    locatePrompt !== null &&\n    'prompt' in locatePrompt\n  ) {\n    const { prompt: innerPrompt, ...rest } = locatePrompt;\n    const hasMultimodalFields = Object.keys(rest).length > 0;\n    normalizedLocatePrompt = hasMultimodalFields ? locatePrompt : innerPrompt;\n  }\n\n  let prompt = normalizedLocatePrompt || opt?.prompt || (opt as any)?.locate; // as a shortcut\n  let deepLocate = false;\n  let cacheable = true;\n  let xpath = undefined;\n\n  if (typeof opt === 'object' && opt !== null) {\n    // Backward-compatible: accept `deepThink` as a deprecated alias for `deepLocate`.\n    // All downstream code works on `deepLocate` only; the compatibility resolution\n    // is intentionally kept here at the entry point so it does not bleed through\n    // the rest of the call stack.\n    deepLocate = opt.deepLocate ?? opt.deepThink ?? false;\n    cacheable = opt.cacheable ?? true;\n    xpath = opt.xpath;\n    if (locatePrompt && opt.prompt && locatePrompt !== opt.prompt) {\n      console.warn(\n        'conflict prompt for item',\n        locatePrompt,\n        opt,\n        'maybe you put the prompt in the wrong place',\n      );\n    }\n    prompt = prompt || opt.prompt;\n  }\n\n  if (!prompt) {\n    debugUtils(\n      'no prompt, will return undefined in buildDetailedLocateParam',\n      opt,\n    );\n    return undefined;\n  }\n\n  const multimodalPrompt = extractMultimodalPrompt(opt);\n  if (multimodalPrompt) {\n    prompt =\n      typeof prompt === 'string'\n        ? {\n            prompt,\n            ...multimodalPrompt,\n          }\n        : {\n            ...prompt,\n            ...multimodalPrompt,\n          };\n  }\n\n  return {\n    prompt,\n    deepLocate,\n    cacheable,\n    xpath,\n  };\n}\n\nexport function buildDetailedLocateParamAndRestParams(\n  locatePrompt: TUserPrompt,\n  opt: LocateOption | undefined,\n  excludeKeys: string[] = [],\n): {\n  locateParam: DetailedLocateParam | undefined;\n  restParams: Record<string, any>;\n} {\n  const multimodalPrompt = extractMultimodalPrompt(opt);\n  const locateParam = buildDetailedLocateParam(locatePrompt, opt);\n\n  // Extract all keys from opt except the ones already included in locateParam\n  const restParams: Record<string, any> = {};\n\n  if (typeof opt === 'object' && opt !== null) {\n    // Get all keys from opt\n    const allKeys = Object.keys(opt);\n\n    // Keys already included in locateParam: prompt, deepLocate, cacheable, xpath\n    const locateParamKeys = Object.keys(locateParam || {});\n    const multimodalPromptKeys =\n      typeof locateParam?.prompt === 'object' && locateParam?.prompt !== null\n        ? Object.keys(multimodalPrompt || {})\n        : [];\n\n    // Extract all other keys\n    for (const key of allKeys) {\n      if (\n        !locateParamKeys.includes(key) &&\n        !multimodalPromptKeys.includes(key) &&\n        !excludeKeys.includes(key) &&\n        key !== 'locate'\n      ) {\n        restParams[key] = opt[key as keyof LocateOption];\n      }\n    }\n  }\n\n  return {\n    locateParam,\n    restParams,\n  };\n}\n"],"names":["debugUtils","getDebug","topLevelTasksPattern","topLevelYamlKeyPattern","webTargetSources","resolveWebTarget","config","entries","source","entry","specifiedTargets","Error","target","explicitMode","mode","interpolateEnvVarRefs","value","keepUnresolvedRefs","result","lastIndex","searchFrom","start","end","rawName","envVar","envValue","process","undefined","multimodalLocateOptionFieldMap","multimodalLocateOptionKeys","Object","extractMultimodalPrompt","opt","key","interpolateEnvVars","content","lines","processedLines","line","trimmedLine","interpolateYamlScriptEnvVars","inTopLevelTasksBlock","indentSize","parseYamlScript","filePath","processedContent","matchedDeviceId","match","deviceId","console","interpolatedContent","obj","yaml","pathTip","assert","Array","buildDetailedLocateParam","locatePrompt","normalizedLocatePrompt","innerPrompt","rest","hasMultimodalFields","prompt","deepLocate","cacheable","xpath","multimodalPrompt","buildDetailedLocateParamAndRestParams","excludeKeys","locateParam","restParams","allKeys","locateParamKeys","multimodalPromptKeys"],"mappings":";;;AAWA,MAAMA,aAAaC,SAAS;AAE5B,MAAMC,uBAAuB;AAC7B,MAAMC,yBAAyB;AAc/B,MAAMC,mBAAsC;IAC1C;IACA;IACA;IACA;CACD;AAEM,SAASC,iBACdC,MAAuB;IAEvB,MAAMC,UAAUH,iBACb,GAAG,CAAC,CAACI,SAAW;YAACA;YAAQF,MAAM,CAACE,OAAO;SAAC,EACxC,MAAM,CACL,CACEC,QAIG,AAAoB,WAAbA,KAAK,CAAC,EAAE;IAGxB,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAChB;IAGF,IAAIA,QAAQ,MAAM,GAAG,GAAG;QACtB,MAAMG,mBAAmBH,QAAQ,GAAG,CAAC,CAAC,CAACC,OAAO,GAAKA;QACnD,MAAM,IAAIG,MACR,CAAC,qEAAqE,EAAED,iBAAiB,IAAI,CAC3F,MACA,4DAA4D,CAAC;IAEnE;IAEA,MAAM,CAACF,QAAQI,OAAO,GAAGL,OAAO,CAAC,EAAE;IACnC,MAAMM,eAAeD,OAAO,IAAI;IAChC,IACE,AAAwB,WAAjBC,gBACPA,AAAiB,WAAjBA,gBACAA,AAAiB,cAAjBA,cAEA,MAAM,IAAIF,MACR,CAAC,wEAAwE,EAAEE,aAAa,EAAE,CAAC;IAI/F,IAAIL,AAAW,WAAXA,UAAqBK,AAAiB,cAAjBA,cACvB,MAAM,IAAIF,MACR;IAIJ,IAAIH,AAAW,cAAXA,UAAwBK,AAAiB,WAAjBA,cAC1B,MAAM,IAAIF,MACR;IAIJ,MAAMG,OACJN,AAAW,WAAXA,SACI,SACAA,AAAW,cAAXA,SACE,YACCK,gBAAgB;IAEzB,IAAIC,AAAS,WAATA,QAAmBF,OAAO,iBAAiB,EAC7C,MAAM,IAAID,MACR;IAIJ,IACEG,AAAS,cAATA,QACA,AAAyC,WAAlCF,OAAO,sBAAsB,EAEpC,MAAM,IAAID,MACR;IAIJ,OAAO;QACLH;QACAM;QACA,QAAQ;YACN,GAAGF,MAAM;YACTE;QACF;IACF;AACF;AAEA,SAASC,sBACPC,KAAa,EACbC,qBAAqB,KAAK;IAE1B,IAAIC,SAAS;IACb,IAAIC,YAAY;IAChB,IAAIC,aAAa;IAEjB,MAAOA,aAAaJ,MAAM,MAAM,CAAE;QAChC,MAAMK,QAAQL,MAAM,OAAO,CAAC,MAAMI;QAClC,IAAIC,AAAU,OAAVA,OACF;QAGF,MAAMC,MAAMN,MAAM,OAAO,CAAC,KAAKK,QAAQ;QACvC,IAAIC,AAAQ,OAARA,KACF;QAGF,MAAMC,UAAUP,MAAM,KAAK,CAACK,QAAQ,GAAGC;QACvC,IAAI,CAACC,SAAS;YACZH,aAAaE,MAAM;YACnB;QACF;QAEAJ,UAAUF,MAAM,KAAK,CAACG,WAAWE;QACjC,MAAMG,SAASD,QAAQ,IAAI;QAC3B,MAAME,WAAWC,QAAQ,GAAG,CAACF,OAAO;QACpC,IAAIC,AAAaE,WAAbF,UACF,IAAIR,oBACFC,UAAUF,MAAM,KAAK,CAACK,OAAOC,MAAM;aAEnC,MAAM,IAAIX,MAAM,CAAC,sBAAsB,EAAEa,OAAO,gBAAgB,CAAC;aAGnEN,UAAUO;QAEZN,YAAYG,MAAM;QAClBF,aAAaE,MAAM;IACrB;IAEA,OAAOJ,SAASF,MAAM,KAAK,CAACG;AAC9B;AAEA,MAAMS,iCAAwE;IAC5E,QAAQ;IACR,yBAAyB;AAC3B;AAEA,MAAMC,6BAA6BC,OAAO,IAAI,CAC5CF;AAGF,SAASG,wBACPC,GAAkB;IAElB,IAAI,AAAe,YAAf,OAAOA,OAAoBA,AAAQ,SAARA,KAC7B;IAGF,MAAMzB,UAAUsB,2BACb,GAAG,CAAC,CAACI,MAAQ;YAACA;YAAKD,GAAG,CAACC,IAAI;SAAC,EAC5B,MAAM,CAAC,CAAC,GAAGjB,MAAM,GAAKA,AAAUW,WAAVX;IAEzB,OAAOT,QAAQ,MAAM,GAChBuB,OAAO,WAAW,CAACvB,WACpBoB;AACN;AAEO,SAASO,mBAAmBC,OAAe;IAEhD,MAAMC,QAAQD,QAAQ,KAAK,CAAC;IAC5B,MAAME,iBAAiBD,MAAM,GAAG,CAAC,CAACE;QAEhC,MAAMC,cAAcD,KAAK,SAAS;QAClC,IAAIC,YAAY,UAAU,CAAC,MAEzB,OAAOD;QAIT,OAAOvB,sBAAsBuB;IAC/B;IAEA,OAAOD,eAAe,IAAI,CAAC;AAC7B;AAEA,SAASG,6BAA6BL,OAAe;IACnD,IAAIM,uBAAuB;IAE3B,OAAON,QACJ,KAAK,CAAC,MACN,GAAG,CAAC,CAACG;QACJ,MAAMC,cAAcD,KAAK,SAAS;QAClC,IAAI,CAACC,eAAeA,YAAY,UAAU,CAAC,MACzC,OAAOD;QAGT,MAAMI,aAAaJ,KAAK,MAAM,GAAGC,YAAY,MAAM;QACnD,IAAIG,AAAe,MAAfA,YAAkB;YACpB,IAAIxC,qBAAqB,IAAI,CAACqC,cAAc;gBAC1CE,uBAAuB;gBACvB,OAAOH;YACT;YAEA,IAAInC,uBAAuB,IAAI,CAACoC,cAC9BE,uBAAuB;QAE3B;QAEA,OAAO1B,sBAAsBuB,MAAMG;IACrC,GACC,IAAI,CAAC;AACV;AAEO,SAASE,gBACdR,OAAe,EACfS,QAAiB;IAEjB,IAAIC,mBAAmBV;IACvB,IAAIA,AAA+B,OAA/BA,QAAQ,OAAO,CAAC,cAAqBA,QAAQ,KAAK,CAAC,sBAAsB;QAC3E,IAAIW;QACJD,mBAAmBV,QAAQ,OAAO,CAChC,sBACA,CAACY,OAAOC;YACNF,kBAAkBE;YAClB,OAAO,CAAC,WAAW,EAAEA,SAAS,CAAC,CAAC;QAClC;QAEFC,QAAQ,IAAI,CACV,CAAC,yEAAyE,EAAEH,gBAAgB,CAAC,CAAC;IAElG;IACA,MAAMI,sBAAsBV,6BAA6BK;IACzD,MAAMM,MAAMC,QAAAA,IAAS,CAACF,qBAAqB;QACzC,QAAQE,QAAAA,WAAgB;IAC1B;IAEA,MAAMC,UAAUT,WAAW,CAAC,iBAAiB,EAAEA,UAAU,GAAG;IAC5DvC,iBAAiB8C;IACjBG,OAAOH,IAAI,KAAK,EAAE,CAAC,4CAA4C,EAAEE,SAAS;IAC1EC,OACEC,MAAM,OAAO,CAACJ,IAAI,KAAK,GACvB,CAAC,0DAA0D,EAAEA,IAAI,KAAK,EAAE;IAE1E,OAAOA;AACT;AAEO,SAASK,yBACdC,YAAyB,EACzBzB,GAAkB;IAElBhC,WAAW,sCAAsCyD,cAAczB;IAI/D,IAAI0B,yBAAsCD;IAC1C,IACE,AAAwB,YAAxB,OAAOA,gBACPA,AAAiB,SAAjBA,gBACA,YAAYA,cACZ;QACA,MAAM,EAAE,QAAQE,WAAW,EAAE,GAAGC,MAAM,GAAGH;QACzC,MAAMI,sBAAsB/B,OAAO,IAAI,CAAC8B,MAAM,MAAM,GAAG;QACvDF,yBAAyBG,sBAAsBJ,eAAeE;IAChE;IAEA,IAAIG,SAASJ,0BAA0B1B,KAAK,UAAWA,KAAa;IACpE,IAAI+B,aAAa;IACjB,IAAIC,YAAY;IAChB,IAAIC;IAEJ,IAAI,AAAe,YAAf,OAAOjC,OAAoBA,AAAQ,SAARA,KAAc;QAK3C+B,aAAa/B,IAAI,UAAU,IAAIA,IAAI,SAAS,IAAI;QAChDgC,YAAYhC,IAAI,SAAS,IAAI;QAC7BiC,QAAQjC,IAAI,KAAK;QACjB,IAAIyB,gBAAgBzB,IAAI,MAAM,IAAIyB,iBAAiBzB,IAAI,MAAM,EAC3DiB,QAAQ,IAAI,CACV,4BACAQ,cACAzB,KACA;QAGJ8B,SAASA,UAAU9B,IAAI,MAAM;IAC/B;IAEA,IAAI,CAAC8B,QAAQ,YACX9D,WACE,gEACAgC;IAKJ,MAAMkC,mBAAmBnC,wBAAwBC;IACjD,IAAIkC,kBACFJ,SACE,AAAkB,YAAlB,OAAOA,SACH;QACEA;QACA,GAAGI,gBAAgB;IACrB,IACA;QACE,GAAGJ,MAAM;QACT,GAAGI,gBAAgB;IACrB;IAGR,OAAO;QACLJ;QACAC;QACAC;QACAC;IACF;AACF;AAEO,SAASE,sCACdV,YAAyB,EACzBzB,GAA6B,EAC7BoC,cAAwB,EAAE;IAK1B,MAAMF,mBAAmBnC,wBAAwBC;IACjD,MAAMqC,cAAcb,yBAAyBC,cAAczB;IAG3D,MAAMsC,aAAkC,CAAC;IAEzC,IAAI,AAAe,YAAf,OAAOtC,OAAoBA,AAAQ,SAARA,KAAc;QAE3C,MAAMuC,UAAUzC,OAAO,IAAI,CAACE;QAG5B,MAAMwC,kBAAkB1C,OAAO,IAAI,CAACuC,eAAe,CAAC;QACpD,MAAMI,uBACJ,AAA+B,YAA/B,OAAOJ,aAAa,UAAuBA,aAAa,WAAW,OAC/DvC,OAAO,IAAI,CAACoC,oBAAoB,CAAC,KACjC,EAAE;QAGR,KAAK,MAAMjC,OAAOsC,QAChB,IACE,CAACC,gBAAgB,QAAQ,CAACvC,QAC1B,CAACwC,qBAAqB,QAAQ,CAACxC,QAC/B,CAACmC,YAAY,QAAQ,CAACnC,QACtBA,AAAQ,aAARA,KAEAqC,UAAU,CAACrC,IAAI,GAAGD,GAAG,CAACC,IAA0B;IAGtD;IAEA,OAAO;QACLoC;QACAC;IACF;AACF"}
+{"version":3,"file":"yaml/utils.mjs","sources":["../../../src/yaml/utils.ts"],"sourcesContent":["import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type {\n  DetailedLocateParam,\n  LocateOption,\n  MidsceneYamlScript,\n  MidsceneYamlScriptWebEnv,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\nimport { buildLocatePromptWithContext } from '../agent/prompt-context';\n\nconst debugUtils = getDebug('yaml:utils');\n\nconst topLevelTasksPattern = /^tasks\\s*:/;\nconst topLevelYamlKeyPattern = /^[^\\s#][^:]*:/;\n\nexport type WebTargetSource = 'page' | 'browser' | 'web' | 'target';\n\nexport type ResolvedWebTarget = {\n  source: WebTargetSource;\n  target: Partial<MidsceneYamlScriptWebEnv> & { mode: 'page' | 'browser' };\n  mode: 'page' | 'browser';\n};\n\nexport type WebTargetConfig = Partial<\n  Record<WebTargetSource, Partial<MidsceneYamlScriptWebEnv>>\n>;\n\nconst webTargetSources: WebTargetSource[] = [\n  'page',\n  'browser',\n  'web',\n  'target',\n];\n\nexport function resolveWebTarget(\n  config: WebTargetConfig,\n): ResolvedWebTarget | undefined {\n  const entries = webTargetSources\n    .map((source) => [source, config[source]] as const)\n    .filter(\n      (\n        entry,\n      ): entry is readonly [\n        WebTargetSource,\n        Partial<MidsceneYamlScriptWebEnv>,\n      ] => typeof entry[1] !== 'undefined',\n    );\n\n  if (entries.length === 0) {\n    return undefined;\n  }\n\n  if (entries.length > 1) {\n    const specifiedTargets = entries.map(([source]) => source);\n    throw new Error(\n      `[midscene] Only one web target can be specified, but found multiple: ${specifiedTargets.join(\n        ', ',\n      )}. Please specify only one of: page, browser, web, or target.`,\n    );\n  }\n\n  const [source, target] = entries[0];\n  const explicitMode = target.mode;\n  if (\n    typeof explicitMode !== 'undefined' &&\n    explicitMode !== 'page' &&\n    explicitMode !== 'browser'\n  ) {\n    throw new Error(\n      `[midscene] web target mode must be either \"page\" or \"browser\", but got \"${explicitMode}\".`,\n    );\n  }\n\n  if (source === 'page' && explicitMode === 'browser') {\n    throw new Error(\n      '[midscene] page target cannot use mode: browser. Use browser: instead.',\n    );\n  }\n\n  if (source === 'browser' && explicitMode === 'page') {\n    throw new Error(\n      '[midscene] browser target cannot use mode: page. Use page: instead.',\n    );\n  }\n\n  const mode =\n    source === 'page'\n      ? 'page'\n      : source === 'browser'\n        ? 'browser'\n        : (explicitMode ?? 'page');\n\n  if (mode === 'page' && target.autoFollowNewPage) {\n    throw new Error(\n      '[midscene] autoFollowNewPage requires browser mode. Use browser: or web.mode: browser.',\n    );\n  }\n\n  if (\n    mode === 'browser' &&\n    typeof target.forceSameTabNavigation !== 'undefined'\n  ) {\n    throw new Error(\n      '[midscene] forceSameTabNavigation cannot be used in browser mode. Use page: or web.mode: page when same-tab navigation is required.',\n    );\n  }\n\n  return {\n    source,\n    mode,\n    target: {\n      ...target,\n      mode,\n    },\n  };\n}\n\nfunction interpolateEnvVarRefs(\n  value: string,\n  keepUnresolvedRefs = false,\n): string {\n  let result = '';\n  let lastIndex = 0;\n  let searchFrom = 0;\n\n  while (searchFrom < value.length) {\n    const start = value.indexOf('${', searchFrom);\n    if (start === -1) {\n      break;\n    }\n\n    const end = value.indexOf('}', start + 2);\n    if (end === -1) {\n      break;\n    }\n\n    const rawName = value.slice(start + 2, end);\n    if (!rawName) {\n      searchFrom = end + 1;\n      continue;\n    }\n\n    result += value.slice(lastIndex, start);\n    const envVar = rawName.trim();\n    const envValue = process.env[envVar];\n    if (envValue === undefined) {\n      if (keepUnresolvedRefs) {\n        result += value.slice(start, end + 1);\n      } else {\n        throw new Error(`Environment variable \"${envVar}\" is not defined`);\n      }\n    } else {\n      result += envValue;\n    }\n    lastIndex = end + 1;\n    searchFrom = end + 1;\n  }\n\n  return result + value.slice(lastIndex);\n}\n\nconst multimodalLocateOptionFieldMap: Record<keyof TMultimodalPrompt, true> = {\n  images: true,\n  convertHttpImage2Base64: true,\n};\n\nconst multimodalLocateOptionKeys = Object.keys(\n  multimodalLocateOptionFieldMap,\n) as Array<keyof TMultimodalPrompt>;\n\nfunction extractMultimodalPrompt(\n  opt?: LocateOption,\n): Partial<TMultimodalPrompt> | undefined {\n  if (typeof opt !== 'object' || opt === null) {\n    return undefined;\n  }\n\n  const entries = multimodalLocateOptionKeys\n    .map((key) => [key, opt[key]] as const)\n    .filter(([, value]) => value !== undefined);\n\n  return entries.length\n    ? (Object.fromEntries(entries) as Partial<TMultimodalPrompt>)\n    : undefined;\n}\n\nexport function interpolateEnvVars(content: string): string {\n  // Process line by line to skip commented lines\n  const lines = content.split('\\n');\n  const processedLines = lines.map((line) => {\n    // Check if the line is a YAML comment (starts with # after optional whitespace)\n    const trimmedLine = line.trimStart();\n    if (trimmedLine.startsWith('#')) {\n      // Skip interpolation for comment lines\n      return line;\n    }\n\n    // Process environment variables for non-comment lines\n    return interpolateEnvVarRefs(line);\n  });\n\n  return processedLines.join('\\n');\n}\n\nfunction interpolateYamlScriptEnvVars(content: string): string {\n  let inTopLevelTasksBlock = false;\n\n  return content\n    .split('\\n')\n    .map((line) => {\n      const trimmedLine = line.trimStart();\n      if (!trimmedLine || trimmedLine.startsWith('#')) {\n        return line;\n      }\n\n      const indentSize = line.length - trimmedLine.length;\n      if (indentSize === 0) {\n        if (topLevelTasksPattern.test(trimmedLine)) {\n          inTopLevelTasksBlock = true;\n          return line;\n        }\n\n        if (topLevelYamlKeyPattern.test(trimmedLine)) {\n          inTopLevelTasksBlock = false;\n        }\n      }\n\n      return interpolateEnvVarRefs(line, inTopLevelTasksBlock);\n    })\n    .join('\\n');\n}\n\nexport function parseYamlScript(\n  content: string,\n  filePath?: string,\n): MidsceneYamlScript {\n  let processedContent = content;\n  if (content.indexOf('android') !== -1 && content.match(/deviceId:\\s*(\\d+)/)) {\n    let matchedDeviceId;\n    processedContent = content.replace(\n      /deviceId:\\s*(\\d+)/g,\n      (match, deviceId) => {\n        matchedDeviceId = deviceId;\n        return `deviceId: '${deviceId}'`;\n      },\n    );\n    console.warn(\n      `please use string-style deviceId in yaml script, for example: deviceId: \"${matchedDeviceId}\"`,\n    );\n  }\n  const interpolatedContent = interpolateYamlScriptEnvVars(processedContent);\n  const obj = yaml.load(interpolatedContent, {\n    schema: yaml.JSON_SCHEMA,\n  }) as MidsceneYamlScript;\n\n  const pathTip = filePath ? `, failed to load ${filePath}` : '';\n  resolveWebTarget(obj);\n  assert(obj.tasks, `property \"tasks\" is required in yaml script ${pathTip}`);\n  assert(\n    Array.isArray(obj.tasks),\n    `property \"tasks\" must be an array in yaml script, but got ${obj.tasks}`,\n  );\n  return obj;\n}\n\nexport function buildDetailedLocateParam(\n  locatePrompt: TUserPrompt,\n  opt?: LocateOption,\n): DetailedLocateParam | undefined {\n  debugUtils('will call buildDetailedLocateParam', locatePrompt, opt);\n  // Normalize object-form TUserPrompt: when the object only contains a\n  // `prompt` string (no multimodal fields like `images`), unwrap it to\n  // avoid double nesting like { prompt: { prompt: '...' } }.\n  let normalizedLocatePrompt: TUserPrompt = locatePrompt;\n  if (\n    typeof locatePrompt === 'object' &&\n    locatePrompt !== null &&\n    'prompt' in locatePrompt\n  ) {\n    const { prompt: innerPrompt, ...rest } = locatePrompt;\n    const hasMultimodalFields = Object.keys(rest).length > 0;\n    normalizedLocatePrompt = hasMultimodalFields ? locatePrompt : innerPrompt;\n  }\n\n  let prompt = normalizedLocatePrompt || opt?.prompt || (opt as any)?.locate; // as a shortcut\n  let deepLocate = false;\n  let cacheable = true;\n  let xpath = undefined;\n\n  if (typeof opt === 'object' && opt !== null) {\n    // Backward-compatible: accept `deepThink` as a deprecated alias for `deepLocate`.\n    // All downstream code works on `deepLocate` only; the compatibility resolution\n    // is intentionally kept here at the entry point so it does not bleed through\n    // the rest of the call stack.\n    deepLocate = opt.deepLocate ?? opt.deepThink ?? false;\n    cacheable = opt.cacheable ?? true;\n    xpath = opt.xpath;\n    if (locatePrompt && opt.prompt && locatePrompt !== opt.prompt) {\n      console.warn(\n        'conflict prompt for item',\n        locatePrompt,\n        opt,\n        'maybe you put the prompt in the wrong place',\n      );\n    }\n    prompt = prompt || opt.prompt;\n  }\n\n  if (!prompt) {\n    debugUtils(\n      'no prompt, will return undefined in buildDetailedLocateParam',\n      opt,\n    );\n    return undefined;\n  }\n\n  const promptDisplay = typeof prompt === 'string' ? prompt : prompt.prompt;\n  const context = opt?.context?.trim() || undefined;\n  prompt = buildLocatePromptWithContext(prompt, opt?.context);\n\n  const multimodalPrompt = extractMultimodalPrompt(opt);\n  if (multimodalPrompt) {\n    prompt =\n      typeof prompt === 'string'\n        ? {\n            prompt,\n            ...multimodalPrompt,\n          }\n        : {\n            ...prompt,\n            ...multimodalPrompt,\n          };\n  }\n\n  return {\n    prompt,\n    ...(context ? { promptDisplay, context } : {}),\n    deepLocate,\n    cacheable,\n    xpath,\n  };\n}\n\nexport function buildDetailedLocateParamAndRestParams(\n  locatePrompt: TUserPrompt,\n  opt: LocateOption | undefined,\n  excludeKeys: string[] = [],\n): {\n  locateParam: DetailedLocateParam | undefined;\n  restParams: Record<string, any>;\n} {\n  const multimodalPrompt = extractMultimodalPrompt(opt);\n  const locateParam = buildDetailedLocateParam(locatePrompt, opt);\n\n  // Extract all keys from opt except the ones already included in locateParam\n  const restParams: Record<string, any> = {};\n\n  if (typeof opt === 'object' && opt !== null) {\n    // Get all keys from opt\n    const allKeys = Object.keys(opt);\n\n    // `context` has already been merged into the locate prompt.\n    const locateParamKeys = Object.keys(locateParam || {});\n    const multimodalPromptKeys =\n      typeof locateParam?.prompt === 'object' && locateParam?.prompt !== null\n        ? Object.keys(multimodalPrompt || {})\n        : [];\n\n    // Extract all other keys\n    for (const key of allKeys) {\n      if (\n        !locateParamKeys.includes(key) &&\n        key !== 'context' &&\n        key !== 'deepThink' &&\n        !multimodalPromptKeys.includes(key) &&\n        !excludeKeys.includes(key) &&\n        key !== 'locate'\n      ) {\n        restParams[key] = opt[key as keyof LocateOption];\n      }\n    }\n  }\n\n  return {\n    locateParam,\n    restParams,\n  };\n}\n"],"names":["debugUtils","getDebug","topLevelTasksPattern","topLevelYamlKeyPattern","webTargetSources","resolveWebTarget","config","entries","source","entry","specifiedTargets","Error","target","explicitMode","mode","interpolateEnvVarRefs","value","keepUnresolvedRefs","result","lastIndex","searchFrom","start","end","rawName","envVar","envValue","process","undefined","multimodalLocateOptionFieldMap","multimodalLocateOptionKeys","Object","extractMultimodalPrompt","opt","key","interpolateEnvVars","content","lines","processedLines","line","trimmedLine","interpolateYamlScriptEnvVars","inTopLevelTasksBlock","indentSize","parseYamlScript","filePath","processedContent","matchedDeviceId","match","deviceId","console","interpolatedContent","obj","yaml","pathTip","assert","Array","buildDetailedLocateParam","locatePrompt","normalizedLocatePrompt","innerPrompt","rest","hasMultimodalFields","prompt","deepLocate","cacheable","xpath","promptDisplay","context","buildLocatePromptWithContext","multimodalPrompt","buildDetailedLocateParamAndRestParams","excludeKeys","locateParam","restParams","allKeys","locateParamKeys","multimodalPromptKeys"],"mappings":";;;;AAYA,MAAMA,aAAaC,SAAS;AAE5B,MAAMC,uBAAuB;AAC7B,MAAMC,yBAAyB;AAc/B,MAAMC,mBAAsC;IAC1C;IACA;IACA;IACA;CACD;AAEM,SAASC,iBACdC,MAAuB;IAEvB,MAAMC,UAAUH,iBACb,GAAG,CAAC,CAACI,SAAW;YAACA;YAAQF,MAAM,CAACE,OAAO;SAAC,EACxC,MAAM,CACL,CACEC,QAIG,AAAoB,WAAbA,KAAK,CAAC,EAAE;IAGxB,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAChB;IAGF,IAAIA,QAAQ,MAAM,GAAG,GAAG;QACtB,MAAMG,mBAAmBH,QAAQ,GAAG,CAAC,CAAC,CAACC,OAAO,GAAKA;QACnD,MAAM,IAAIG,MACR,CAAC,qEAAqE,EAAED,iBAAiB,IAAI,CAC3F,MACA,4DAA4D,CAAC;IAEnE;IAEA,MAAM,CAACF,QAAQI,OAAO,GAAGL,OAAO,CAAC,EAAE;IACnC,MAAMM,eAAeD,OAAO,IAAI;IAChC,IACE,AAAwB,WAAjBC,gBACPA,AAAiB,WAAjBA,gBACAA,AAAiB,cAAjBA,cAEA,MAAM,IAAIF,MACR,CAAC,wEAAwE,EAAEE,aAAa,EAAE,CAAC;IAI/F,IAAIL,AAAW,WAAXA,UAAqBK,AAAiB,cAAjBA,cACvB,MAAM,IAAIF,MACR;IAIJ,IAAIH,AAAW,cAAXA,UAAwBK,AAAiB,WAAjBA,cAC1B,MAAM,IAAIF,MACR;IAIJ,MAAMG,OACJN,AAAW,WAAXA,SACI,SACAA,AAAW,cAAXA,SACE,YACCK,gBAAgB;IAEzB,IAAIC,AAAS,WAATA,QAAmBF,OAAO,iBAAiB,EAC7C,MAAM,IAAID,MACR;IAIJ,IACEG,AAAS,cAATA,QACA,AAAyC,WAAlCF,OAAO,sBAAsB,EAEpC,MAAM,IAAID,MACR;IAIJ,OAAO;QACLH;QACAM;QACA,QAAQ;YACN,GAAGF,MAAM;YACTE;QACF;IACF;AACF;AAEA,SAASC,sBACPC,KAAa,EACbC,qBAAqB,KAAK;IAE1B,IAAIC,SAAS;IACb,IAAIC,YAAY;IAChB,IAAIC,aAAa;IAEjB,MAAOA,aAAaJ,MAAM,MAAM,CAAE;QAChC,MAAMK,QAAQL,MAAM,OAAO,CAAC,MAAMI;QAClC,IAAIC,AAAU,OAAVA,OACF;QAGF,MAAMC,MAAMN,MAAM,OAAO,CAAC,KAAKK,QAAQ;QACvC,IAAIC,AAAQ,OAARA,KACF;QAGF,MAAMC,UAAUP,MAAM,KAAK,CAACK,QAAQ,GAAGC;QACvC,IAAI,CAACC,SAAS;YACZH,aAAaE,MAAM;YACnB;QACF;QAEAJ,UAAUF,MAAM,KAAK,CAACG,WAAWE;QACjC,MAAMG,SAASD,QAAQ,IAAI;QAC3B,MAAME,WAAWC,QAAQ,GAAG,CAACF,OAAO;QACpC,IAAIC,AAAaE,WAAbF,UACF,IAAIR,oBACFC,UAAUF,MAAM,KAAK,CAACK,OAAOC,MAAM;aAEnC,MAAM,IAAIX,MAAM,CAAC,sBAAsB,EAAEa,OAAO,gBAAgB,CAAC;aAGnEN,UAAUO;QAEZN,YAAYG,MAAM;QAClBF,aAAaE,MAAM;IACrB;IAEA,OAAOJ,SAASF,MAAM,KAAK,CAACG;AAC9B;AAEA,MAAMS,iCAAwE;IAC5E,QAAQ;IACR,yBAAyB;AAC3B;AAEA,MAAMC,6BAA6BC,OAAO,IAAI,CAC5CF;AAGF,SAASG,wBACPC,GAAkB;IAElB,IAAI,AAAe,YAAf,OAAOA,OAAoBA,AAAQ,SAARA,KAC7B;IAGF,MAAMzB,UAAUsB,2BACb,GAAG,CAAC,CAACI,MAAQ;YAACA;YAAKD,GAAG,CAACC,IAAI;SAAC,EAC5B,MAAM,CAAC,CAAC,GAAGjB,MAAM,GAAKA,AAAUW,WAAVX;IAEzB,OAAOT,QAAQ,MAAM,GAChBuB,OAAO,WAAW,CAACvB,WACpBoB;AACN;AAEO,SAASO,mBAAmBC,OAAe;IAEhD,MAAMC,QAAQD,QAAQ,KAAK,CAAC;IAC5B,MAAME,iBAAiBD,MAAM,GAAG,CAAC,CAACE;QAEhC,MAAMC,cAAcD,KAAK,SAAS;QAClC,IAAIC,YAAY,UAAU,CAAC,MAEzB,OAAOD;QAIT,OAAOvB,sBAAsBuB;IAC/B;IAEA,OAAOD,eAAe,IAAI,CAAC;AAC7B;AAEA,SAASG,6BAA6BL,OAAe;IACnD,IAAIM,uBAAuB;IAE3B,OAAON,QACJ,KAAK,CAAC,MACN,GAAG,CAAC,CAACG;QACJ,MAAMC,cAAcD,KAAK,SAAS;QAClC,IAAI,CAACC,eAAeA,YAAY,UAAU,CAAC,MACzC,OAAOD;QAGT,MAAMI,aAAaJ,KAAK,MAAM,GAAGC,YAAY,MAAM;QACnD,IAAIG,AAAe,MAAfA,YAAkB;YACpB,IAAIxC,qBAAqB,IAAI,CAACqC,cAAc;gBAC1CE,uBAAuB;gBACvB,OAAOH;YACT;YAEA,IAAInC,uBAAuB,IAAI,CAACoC,cAC9BE,uBAAuB;QAE3B;QAEA,OAAO1B,sBAAsBuB,MAAMG;IACrC,GACC,IAAI,CAAC;AACV;AAEO,SAASE,gBACdR,OAAe,EACfS,QAAiB;IAEjB,IAAIC,mBAAmBV;IACvB,IAAIA,AAA+B,OAA/BA,QAAQ,OAAO,CAAC,cAAqBA,QAAQ,KAAK,CAAC,sBAAsB;QAC3E,IAAIW;QACJD,mBAAmBV,QAAQ,OAAO,CAChC,sBACA,CAACY,OAAOC;YACNF,kBAAkBE;YAClB,OAAO,CAAC,WAAW,EAAEA,SAAS,CAAC,CAAC;QAClC;QAEFC,QAAQ,IAAI,CACV,CAAC,yEAAyE,EAAEH,gBAAgB,CAAC,CAAC;IAElG;IACA,MAAMI,sBAAsBV,6BAA6BK;IACzD,MAAMM,MAAMC,QAAAA,IAAS,CAACF,qBAAqB;QACzC,QAAQE,QAAAA,WAAgB;IAC1B;IAEA,MAAMC,UAAUT,WAAW,CAAC,iBAAiB,EAAEA,UAAU,GAAG;IAC5DvC,iBAAiB8C;IACjBG,OAAOH,IAAI,KAAK,EAAE,CAAC,4CAA4C,EAAEE,SAAS;IAC1EC,OACEC,MAAM,OAAO,CAACJ,IAAI,KAAK,GACvB,CAAC,0DAA0D,EAAEA,IAAI,KAAK,EAAE;IAE1E,OAAOA;AACT;AAEO,SAASK,yBACdC,YAAyB,EACzBzB,GAAkB;IAElBhC,WAAW,sCAAsCyD,cAAczB;IAI/D,IAAI0B,yBAAsCD;IAC1C,IACE,AAAwB,YAAxB,OAAOA,gBACPA,AAAiB,SAAjBA,gBACA,YAAYA,cACZ;QACA,MAAM,EAAE,QAAQE,WAAW,EAAE,GAAGC,MAAM,GAAGH;QACzC,MAAMI,sBAAsB/B,OAAO,IAAI,CAAC8B,MAAM,MAAM,GAAG;QACvDF,yBAAyBG,sBAAsBJ,eAAeE;IAChE;IAEA,IAAIG,SAASJ,0BAA0B1B,KAAK,UAAWA,KAAa;IACpE,IAAI+B,aAAa;IACjB,IAAIC,YAAY;IAChB,IAAIC;IAEJ,IAAI,AAAe,YAAf,OAAOjC,OAAoBA,AAAQ,SAARA,KAAc;QAK3C+B,aAAa/B,IAAI,UAAU,IAAIA,IAAI,SAAS,IAAI;QAChDgC,YAAYhC,IAAI,SAAS,IAAI;QAC7BiC,QAAQjC,IAAI,KAAK;QACjB,IAAIyB,gBAAgBzB,IAAI,MAAM,IAAIyB,iBAAiBzB,IAAI,MAAM,EAC3DiB,QAAQ,IAAI,CACV,4BACAQ,cACAzB,KACA;QAGJ8B,SAASA,UAAU9B,IAAI,MAAM;IAC/B;IAEA,IAAI,CAAC8B,QAAQ,YACX9D,WACE,gEACAgC;IAKJ,MAAMkC,gBAAgB,AAAkB,YAAlB,OAAOJ,SAAsBA,SAASA,OAAO,MAAM;IACzE,MAAMK,UAAUnC,KAAK,SAAS,UAAUL;IACxCmC,SAASM,6BAA6BN,QAAQ9B,KAAK;IAEnD,MAAMqC,mBAAmBtC,wBAAwBC;IACjD,IAAIqC,kBACFP,SACE,AAAkB,YAAlB,OAAOA,SACH;QACEA;QACA,GAAGO,gBAAgB;IACrB,IACA;QACE,GAAGP,MAAM;QACT,GAAGO,gBAAgB;IACrB;IAGR,OAAO;QACLP;QACA,GAAIK,UAAU;YAAED;YAAeC;QAAQ,IAAI,CAAC,CAAC;QAC7CJ;QACAC;QACAC;IACF;AACF;AAEO,SAASK,sCACdb,YAAyB,EACzBzB,GAA6B,EAC7BuC,cAAwB,EAAE;IAK1B,MAAMF,mBAAmBtC,wBAAwBC;IACjD,MAAMwC,cAAchB,yBAAyBC,cAAczB;IAG3D,MAAMyC,aAAkC,CAAC;IAEzC,IAAI,AAAe,YAAf,OAAOzC,OAAoBA,AAAQ,SAARA,KAAc;QAE3C,MAAM0C,UAAU5C,OAAO,IAAI,CAACE;QAG5B,MAAM2C,kBAAkB7C,OAAO,IAAI,CAAC0C,eAAe,CAAC;QACpD,MAAMI,uBACJ,AAA+B,YAA/B,OAAOJ,aAAa,UAAuBA,aAAa,WAAW,OAC/D1C,OAAO,IAAI,CAACuC,oBAAoB,CAAC,KACjC,EAAE;QAGR,KAAK,MAAMpC,OAAOyC,QAChB,IACE,CAACC,gBAAgB,QAAQ,CAAC1C,QAC1BA,AAAQ,cAARA,OACAA,AAAQ,gBAARA,OACA,CAAC2C,qBAAqB,QAAQ,CAAC3C,QAC/B,CAACsC,YAAY,QAAQ,CAACtC,QACtBA,AAAQ,aAARA,KAEAwC,UAAU,CAACxC,IAAI,GAAGD,GAAG,CAACC,IAA0B;IAGtD;IAEA,OAAO;QACLuC;QACAC;IACF;AACF"}

package/dist/lib/agent/agent.js

@@ -54,6 +54,7 @@ const logger_namespaceObject = require("@midscene/shared/logger");
 const utils_namespaceObject = require("@midscene/shared/utils");
 const external_device_index_js_namespaceObject = require("../device/index.js");
 const external_cache_config_js_namespaceObject = require("./cache-config.js");
+const external_file_chooser_js_namespaceObject = require("./file-chooser.js");
 const external_metrics_js_namespaceObject = require("./metrics.js");
 const external_progress_index_js_namespaceObject = require("./progress/index.js");
 const external_prompt_context_js_namespaceObject = require("./prompt-context.js");
@@ -154,7 +155,7 @@ class Agent {
             openFrameSource: async ()=>await this.interface.openFrameSource?.() ?? void 0,
             screenshot: ()=>this.interface.screenshotBase64(),
             captureRepresentative: ()=>this.getUIContext('assert'),
-            runAssert: (assertion, uiContext, msg, assertOpt)=>this.aiAssertWithContext(assertion, uiContext, msg, assertOpt),
+            runAssert: (assertion, uiContext, msg, assertOpt)=>this.aiAssertWithUIContext(assertion, uiContext, msg, assertOpt),
             runBoolean: (prompt, uiContext, boolOpt)=>this.aiBooleanWithContext(prompt, uiContext, boolOpt),
             onStopped: ()=>{
                 this.activeObserver = null;
@@ -327,13 +328,13 @@ class Agent {
         }
         (0, utils_namespaceObject.assert)('string' == typeof value || 'number' == typeof value, 'input value must be a string or number, use empty string if you want to clear the input');
         (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for input');
-        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt, opt);
         const stringValue = 'number' == typeof value ? String(value) : value;
         const mode = opt?.mode === 'append' ? 'typeOnly' : opt?.mode;
         await this.callActionInActionSpace('Input', {
-            ...opt || {},
+            ...restParams,
             value: stringValue,
-            locate: detailedLocateParam,
+            locate: locateParam,
             mode
         });
     }
@@ -353,10 +354,10 @@ class Agent {
             };
         }
         (0, utils_namespaceObject.assert)(opt?.keyName, 'missing keyName for keyboard press');
-        const detailedLocateParam = locatePrompt ? (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt) : void 0;
+        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt || '', opt);
         await this.callActionInActionSpace('KeyboardPress', {
-            ...opt || {},
-            locate: detailedLocateParam
+            ...restParams,
+            locate: locateParam
         });
     }
     async aiScroll(locatePromptOrScrollParam, locatePromptOrOpt, optOrUndefined) {
@@ -385,25 +386,25 @@ class Agent {
                 scrollType: normalizedScrollType
             };
         }
-        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt || '', opt);
+        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt || '', opt);
         await this.callActionInActionSpace('Scroll', {
-            ...opt || {},
-            locate: detailedLocateParam
+            ...restParams,
+            locate: locateParam
         });
     }
     async aiPinch(locatePrompt, opt) {
-        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt || '', opt);
+        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt || '', opt);
         await this.callActionInActionSpace('Pinch', {
-            ...opt,
-            locate: detailedLocateParam
+            ...restParams,
+            locate: locateParam
         });
     }
     async aiLongPress(locatePrompt, opt) {
         (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for long press');
-        const detailedLocateParam = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        const { locateParam, restParams } = (0, external_yaml_index_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePrompt, opt);
         await this.callActionInActionSpace('LongPress', {
-            ...opt || {},
-            locate: detailedLocateParam
+            ...restParams,
+            locate: locateParam
         });
     }
     async aiClearInput(locatePrompt, opt) {
@@ -459,7 +460,7 @@ class Agent {
                 }
             }
             const imagesIncludeCount = deepThink ? 2 : 1;
-            const { output: actionOutput } = await this.taskExecutor.action(taskPrompt, planningModel, defaultModel, includeLocateInPlanning, aiActContext, cacheable, replanningCycleLimit, imagesIncludeCount, deepThink, fileChooserAccept, deepLocate, abortSignal, internalReportDisplay);
+            const { output: actionOutput } = await this.taskExecutor.action(taskPrompt, planningModel, defaultModel, includeLocateInPlanning, aiActContext, cacheable, replanningCycleLimit, imagesIncludeCount, deepThink, void 0, deepLocate, abortSignal, internalReportDisplay);
             if (this.taskCache && false !== cacheable) {
                 const yamlFlow = cachedYamlFailed ? [] : actionOutput?.yamlFlow;
                 if (!cachedYamlFailed && !yamlFlow?.length) return actionOutput?.output;
@@ -481,7 +482,34 @@ class Agent {
             }
             return actionOutput?.output;
         };
-        return await runAiAct();
+        const fileChooserAccepter = this.interface.registerFileChooserListener ? new external_file_chooser_js_namespaceObject.FileChooserAccepter(this.interface) : void 0;
+        this.activeFileChooserAccepter = fileChooserAccepter;
+        this.activeFileChooserAllowedDir = opt?.fileChooserAllowedDir ? (0, external_node_path_namespaceObject.resolve)(opt.fileChooserAllowedDir) : void 0;
+        let aiActError;
+        let fileChooserHandlingError;
+        let result;
+        try {
+            if (fileChooserAccept?.length) {
+                if (!fileChooserAccepter) throw new Error(`File upload is not supported on ${this.interface.interfaceType}`);
+                await fileChooserAccepter.register(fileChooserAccept);
+            }
+            result = await runAiAct();
+        } catch (error) {
+            aiActError = {
+                error
+            };
+        } finally{
+            this.activeFileChooserAccepter = void 0;
+            this.activeFileChooserAllowedDir = void 0;
+            try {
+                fileChooserHandlingError = await fileChooserAccepter?.clear();
+            } catch (error) {
+                warn(`Failed to clear file chooser registration: ${error}`);
+            }
+        }
+        if (aiActError) throw aiActError.error;
+        if (fileChooserHandlingError) throw fileChooserHandlingError;
+        return result;
     }
     async runMarkdown(markdownPath, opt) {
         const markdown = await (0, promises_namespaceObject.readFile)(markdownPath, 'utf-8');
@@ -551,16 +579,18 @@ class Agent {
         };
     }
     async aiAssert(assertion, msg, opt) {
-        return this.aiAssertWithContext(assertion, void 0, msg, opt);
+        return this.aiAssertWithUIContext(assertion, void 0, msg, opt);
     }
-    async aiAssertWithContext(assertion, uiContext, msg, opt) {
+    async aiAssertWithUIContext(assertion, uiContext, msg, opt) {
         const modelRuntime = this.resolveModelRuntime('insight');
         const serviceOpt = {
             domIncluded: opt?.domIncluded ?? defaultServiceExtractOption.domIncluded,
-            screenshotIncluded: opt?.screenshotIncluded ?? defaultServiceExtractOption.screenshotIncluded
+            screenshotIncluded: opt?.screenshotIncluded ?? defaultServiceExtractOption.screenshotIncluded,
+            ...opt?.context !== void 0 ? {
+                context: opt.context
+            } : {}
         };
-        const assertionWithContext = (0, external_prompt_context_js_namespaceObject.buildPromptWithContext)(assertion, opt?.context);
-        const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(assertionWithContext);
+        const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(assertion);
         const assertionText = 'string' == typeof assertion ? assertion : assertion.prompt;
         const executionOptions = {
             abortSignal: opt?.abortSignal,
@@ -858,6 +888,8 @@ class Agent {
         _define_property(this, "activeObserver", null);
         _define_property(this, "executionDumpIndexByRunner", new WeakMap());
         _define_property(this, "fullActionSpace", void 0);
+        _define_property(this, "activeFileChooserAccepter", void 0);
+        _define_property(this, "activeFileChooserAllowedDir", void 0);
         _define_property(this, "reportGenerator", void 0);
         _define_property(this, "lastExecutionDump", void 0);
         this.interface = interfaceInstance;
@@ -886,8 +918,16 @@ class Agent {
             cacheDir: cacheConfigObj.cacheDir
         });
         const baseActionSpace = this.interface.actionSpace();
+        const fileChooserActions = this.interface.registerFileChooserListener ? [
+            (0, external_device_index_js_namespaceObject.defineActionRegisterFileChooserAccept)(async (files)=>{
+                if (!this.activeFileChooserAccepter) throw new Error('RegisterFileChooserAccept can only be used while aiAct is running');
+                if (!this.activeFileChooserAllowedDir) throw new Error('RegisterFileChooserAccept requires aiAct option fileChooserAllowedDir');
+                await this.activeFileChooserAccepter.registerFromAllowedDir(files, this.activeFileChooserAllowedDir);
+            })
+        ] : [];
         this.fullActionSpace = [
             ...baseActionSpace,
+            ...fileChooserActions,
             (0, external_device_index_js_namespaceObject.defineActionSleep)()
         ];
         this.taskExecutor = new external_tasks_js_namespaceObject.TaskExecutor(this.interface, this.service, {