@midscene/core 1.10.8-beta-20260724035136.0 → 1.10.8

package/dist/lib/agent/ui-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/ui-utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TUserPrompt } from '@/common';\nimport { userPromptToString } from '@/common';\nimport type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport type TaskTitleType =\n  | 'Tap'\n  | 'Hover'\n  | 'Input'\n  | 'RightClick'\n  | 'KeyboardPress'\n  | 'Scroll'\n  | 'Act'\n  | 'Query'\n  | 'Assert'\n  | 'WaitFor'\n  | 'Locate'\n  | 'Markdown'\n  | 'Boolean'\n  | 'Number'\n  | 'String';\n\nexport function taskTitleStr(type: TaskTitleType, prompt: string) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      // Prefer AI-generated output.log over user input\n      const planTask = task as ExecutionTaskPlanning;\n      value =\n        planTask.output?.log ||\n        planTask.param?.userInstructionDisplay ||\n        (planTask.param?.userInstruction\n          ? userPromptToString(planTask.param.userInstruction as TUserPrompt)\n          : undefined);\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'PullGesture'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    // Flatten `{key: \"raw value\"}` into `key: raw value` instead of emitting\n    // a pretty-printed JSON string. JSON.stringify would escape every inner\n    // quote as `\\\"`, and the UI renders the result as plain text — so a\n    // command like `grep -E \"version\"` ends up shown with the literal\n    // backslashes, which is both noisy and confusing for users.\n    const entries = Object.entries(value as Record<string, unknown>);\n    if (entries.length === 0) {\n      return '';\n    }\n    const formatValue = (v: unknown): string => {\n      if (typeof v === 'string') return v;\n      if (v === null || v === undefined) return String(v);\n      if (typeof v === 'object') return JSON.stringify(v);\n      return String(v);\n    };\n    if (entries.length === 1) {\n      const [key, v] = entries[0];\n      return `${key}: ${formatValue(v)}`;\n    }\n    return entries.map(([key, v]) => `${key}: ${formatValue(v)}`).join(', ');\n  }\n\n  return String(value);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","planTask","userPromptToString","locateStr","entries","formatValue","v","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACQO,SAASI,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAE9B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC,SACdA,QAAQ,kBAAkB,UAC/BC,MAAM,OAAO,CAACD,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BE;IAIN,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIN,UAAU,MAAM,EAClB,OAAO;QACL,SAASK,UAAUL,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASK,UAAUL,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAEQ,UAAU,EAAE,GAAGR;QAEvB,IAAI,AAAsB,YAAtB,OAAOQ,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQP,cAAcO;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAmBO,SAASC,aAAaC,IAAmB,EAAEjB,MAAc;IAC9D,IAAIA,QACF,OAAO,GAAGiB,KAAK,GAAG,EAAEjB,QAAQ;IAE9B,OAAOiB;AACT;AAEO,SAASC,SAASrB,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,MAAsC;SACzD;QAEL,MAAMsB,WAAWtB;QACjBgB,QACEM,SAAS,MAAM,EAAE,OACjBA,SAAS,KAAK,EAAE,0BACfA,CAAAA,SAAS,KAAK,EAAE,kBACbC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBD,SAAS,KAAK,CAAC,eAAe,IACjDR,MAAQ;IAChB;IAGF,IAAId,AAAc,cAAdA,KAAK,IAAI,EACXgB,QAAQP,oBAAqBT,MAAc,OAAO,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;QAChC,MAAME,SAAUF,MAA8B,OAAO;QACrD,MAAMwB,YAAYtB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,MAA8B,OAAO,QAC/CgB,QAAQ,GAAIhB,MAA8B,OAAO,OAAO,EAAE,CAAC;aACtD,IACL,AAA4D,YAA5D,OAAQA,MAA8B,OAAO,YAE7CgB,QAAQZ,eAAgBJ,MAA8B;aACjD,IACL,AAA2D,YAA3D,OAAQA,MAA8B,OAAO,aAC5CA,MAA8B,YAAY,eAE3CgB,QAAQV,aAAcN,MAA8B;aAC/C,IACL,AAAuD,WAA/CA,MAA8B,OAAO,OAE7CgB,QAAShB,MAA8B,OAAO;aACzC,IACJA,MAA8B,SAC/B,AAAgD,YAAhD,OAAQA,MAA8B,SACtCL,OAAO,IAAI,CAAEK,MAA8B,SAAS,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,MAA8B;QAGzC,IAAIwB,WAEAR,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGQ,UAAU,GAAG,EAAER,OAAO,GAEzBQ;IAGd;IAEA,IAAI,AAAiB,WAAVR,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMQ,YAAYvB,eAAee;QACjC,IAAIQ,WACF,OAAOA;QAOT,MAAMC,UAAU9B,OAAO,OAAO,CAACqB;QAC/B,IAAIS,AAAmB,MAAnBA,QAAQ,MAAM,EAChB,OAAO;QAET,MAAMC,cAAc,CAACC;YACnB,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOA;YAClC,IAAIA,QAAAA,GAA+B,OAAOC,OAAOD;YACjD,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOV,KAAK,SAAS,CAACU;YACjD,OAAOC,OAAOD;QAChB;QACA,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;YACxB,MAAM,CAAC/B,KAAKiC,EAAE,GAAGF,OAAO,CAAC,EAAE;YAC3B,OAAO,GAAG/B,IAAI,EAAE,EAAEgC,YAAYC,IAAI;QACpC;QACA,OAAOF,QAAQ,GAAG,CAAC,CAAC,CAAC/B,KAAKiC,EAAE,GAAK,GAAGjC,IAAI,EAAE,EAAEgC,YAAYC,IAAI,EAAE,IAAI,CAAC;IACrE;IAEA,OAAOC,OAAOZ;AAChB"}
+{"version":3,"file":"agent/ui-utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/ui-utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TUserPrompt } from '@/common';\nimport { userPromptToString } from '@/common';\nimport type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.promptDisplay === 'string') {\n      return locate.promptDisplay;\n    }\n\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport type TaskTitleType =\n  | 'Tap'\n  | 'Hover'\n  | 'Input'\n  | 'RightClick'\n  | 'KeyboardPress'\n  | 'Scroll'\n  | 'Act'\n  | 'Query'\n  | 'Assert'\n  | 'WaitFor'\n  | 'Locate'\n  | 'Markdown'\n  | 'Boolean'\n  | 'Number'\n  | 'String';\n\nexport function taskTitleStr(type: TaskTitleType, prompt: string) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      // Prefer AI-generated output.log over user input\n      const planTask = task as ExecutionTaskPlanning;\n      value =\n        planTask.output?.log ||\n        planTask.param?.userInstructionDisplay ||\n        (planTask.param?.userInstruction\n          ? userPromptToString(planTask.param.userInstruction as TUserPrompt)\n          : undefined);\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'PullGesture'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    // Flatten `{key: \"raw value\"}` into `key: raw value` instead of emitting\n    // a pretty-printed JSON string. JSON.stringify would escape every inner\n    // quote as `\\\"`, and the UI renders the result as plain text — so a\n    // command like `grep -E \"version\"` ends up shown with the literal\n    // backslashes, which is both noisy and confusing for users.\n    const entries = Object.entries(value as Record<string, unknown>);\n    if (entries.length === 0) {\n      return '';\n    }\n    const formatValue = (v: unknown): string => {\n      if (typeof v === 'string') return v;\n      if (v === null || v === undefined) return String(v);\n      if (typeof v === 'object') return JSON.stringify(v);\n      return String(v);\n    };\n    if (entries.length === 1) {\n      const [key, v] = entries[0];\n      return `${key}: ${formatValue(v)}`;\n    }\n    return entries.map(([key, v]) => `${key}: ${formatValue(v)}`).join(', ');\n  }\n\n  return String(value);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","planTask","userPromptToString","locateStr","entries","formatValue","v","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACQO,SAASI,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAgC,YAAhC,OAAOA,OAAO,aAAa,EAC7B,OAAOA,OAAO,aAAa;QAI7B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC,SACdA,QAAQ,kBAAkB,UAC/BC,MAAM,OAAO,CAACD,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BE;IAIN,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIN,UAAU,MAAM,EAClB,OAAO;QACL,SAASK,UAAUL,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASK,UAAUL,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAEQ,UAAU,EAAE,GAAGR;QAEvB,IAAI,AAAsB,YAAtB,OAAOQ,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQP,cAAcO;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAmBO,SAASC,aAAaC,IAAmB,EAAEjB,MAAc;IAC9D,IAAIA,QACF,OAAO,GAAGiB,KAAK,GAAG,EAAEjB,QAAQ;IAE9B,OAAOiB;AACT;AAEO,SAASC,SAASrB,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,MAAsC;SACzD;QAEL,MAAMsB,WAAWtB;QACjBgB,QACEM,SAAS,MAAM,EAAE,OACjBA,SAAS,KAAK,EAAE,0BACfA,CAAAA,SAAS,KAAK,EAAE,kBACbC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBD,SAAS,KAAK,CAAC,eAAe,IACjDR,MAAQ;IAChB;IAGF,IAAId,AAAc,cAAdA,KAAK,IAAI,EACXgB,QAAQP,oBAAqBT,MAAc,OAAO,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;QAChC,MAAME,SAAUF,MAA8B,OAAO;QACrD,MAAMwB,YAAYtB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,MAA8B,OAAO,QAC/CgB,QAAQ,GAAIhB,MAA8B,OAAO,OAAO,EAAE,CAAC;aACtD,IACL,AAA4D,YAA5D,OAAQA,MAA8B,OAAO,YAE7CgB,QAAQZ,eAAgBJ,MAA8B;aACjD,IACL,AAA2D,YAA3D,OAAQA,MAA8B,OAAO,aAC5CA,MAA8B,YAAY,eAE3CgB,QAAQV,aAAcN,MAA8B;aAC/C,IACL,AAAuD,WAA/CA,MAA8B,OAAO,OAE7CgB,QAAShB,MAA8B,OAAO;aACzC,IACJA,MAA8B,SAC/B,AAAgD,YAAhD,OAAQA,MAA8B,SACtCL,OAAO,IAAI,CAAEK,MAA8B,SAAS,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,MAA8B;QAGzC,IAAIwB,WAEAR,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGQ,UAAU,GAAG,EAAER,OAAO,GAEzBQ;IAGd;IAEA,IAAI,AAAiB,WAAVR,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMQ,YAAYvB,eAAee;QACjC,IAAIQ,WACF,OAAOA;QAOT,MAAMC,UAAU9B,OAAO,OAAO,CAACqB;QAC/B,IAAIS,AAAmB,MAAnBA,QAAQ,MAAM,EAChB,OAAO;QAET,MAAMC,cAAc,CAACC;YACnB,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOA;YAClC,IAAIA,QAAAA,GAA+B,OAAOC,OAAOD;YACjD,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOV,KAAK,SAAS,CAACU;YACjD,OAAOC,OAAOD;QAChB;QACA,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;YACxB,MAAM,CAAC/B,KAAKiC,EAAE,GAAGF,OAAO,CAAC,EAAE;YAC3B,OAAO,GAAG/B,IAAI,EAAE,EAAEgC,YAAYC,IAAI;QACpC;QACA,OAAOF,QAAQ,GAAG,CAAC,CAAC,CAAC/B,KAAKiC,EAAE,GAAK,GAAGjC,IAAI,EAAE,EAAEgC,YAAYC,IAAI,EAAE,IAAI,CAAC;IACrE;IAEA,OAAOC,OAAOZ;AAChB"}

package/dist/lib/agent/utils.js

@@ -232,7 +232,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.10.8-beta-20260724035136.0";
+const getMidsceneVersion = ()=>"1.10.8";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/ai-model/inspect.js

@@ -381,7 +381,7 @@ async function AiExtractElementInfo(options) {
         referenceImagesIncluded: !!multimodalPrompt?.images?.length
     });
     const screenshotBase64 = context.screenshot.base64;
-    const extractDataPromptText = (0, extraction_js_namespaceObject.extractDataQueryPrompt)(options.pageDescription || '', dataQuery);
+    const extractDataPromptText = (0, extraction_js_namespaceObject.extractDataQueryPrompt)(options.pageDescription || '', dataQuery, extractOption?.context);
     const userContent = [];
     if (extractOption?.screenshotIncluded !== false) {
         const screenshotSequence = context.screenshotSequence;

package/dist/lib/ai-model/inspect.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/inspect.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequestContext,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\ntype SectionLocateObjectResponse = Awaited<\n  ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n>;\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const referenceImageMessages =\n    typeof targetElementDescription === 'string'\n      ? undefined\n      : await multimodalPromptToChatMessages(\n          userPromptToMultimodalPrompt(targetElementDescription),\n        );\n  const locateRequest: LocateRequestContext = {\n    elementDescriptionText: userPromptToString(targetElementDescription),\n    locateImage,\n    referenceImageMessages,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(\n    targetElementDescription,\n    locateOptions,\n    locateRequest,\n  );\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugInspect('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          locateRequest.elementDescriptionText,\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  _elementDescription: TUserPrompt,\n  options: LocateOptions,\n  locateRequest: LocateRequestContext,\n): Promise<LocateModelResponse> {\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const userInstructionPrompt = findElementPrompt(\n    locateRequest.elementDescriptionText,\n  );\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateRequest.locateImage.imageBase64,\n    width: locateRequest.locateImage.width,\n    height: locateRequest.locateImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (locateRequest.referenceImageMessages) {\n    msgs.push(...locateRequest.referenceImageMessages);\n  }\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AIElementLocateResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'locate',\n          retryTimes: modelRuntime.config.retryCount,\n          retryInterval: modelRuntime.config.retryInterval,\n        }),\n      parseResponse: (response): LocateModelResponse => {\n        const rawResponse = response.contentString;\n        const errors: string[] | undefined =\n          'errors' in response.content ? response.content.errors : [];\n        if (\n          !hasLocateResult(response.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: errors as string[],\n          };\n        }\n\n        const locatedPixelBbox =\n          resultAdapter.adaptElementLocateResultToPixelBbox(response.content, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n        return {\n          locatedPixelBbox,\n          rawResponse,\n          rawChoiceMessage: response.rawChoiceMessage,\n          usage: response.usage,\n          reasoningContent: response.reasoning_content,\n          errors: errors as string[],\n        };\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        const modelErrors =\n          'errors' in response.content ? response.content.errors : undefined;\n        const message =\n          modelErrors && modelErrors.length > 0\n            ? `${modelErrors.join('\\n')} (${parseErrorMessage})`\n            : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          response.contentString,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugInspect(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let parsedResult:\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AISectionLocatorResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'section-locator',\n          retryTimes: modelRuntime.config.retryCount,\n          retryInterval: modelRuntime.config.retryInterval,\n        }),\n      parseResponse: (result) => {\n        const sectionError = result.content.error;\n        if (\n          !hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return { result, sectionError };\n        }\n\n        const adaptedResult =\n          resultAdapter.adaptSectionLocateResultToPixelBboxGroup(\n            result.content,\n            {\n              preparedSize: preparedImage.preparedSize,\n              contentSize: preparedImage.contentSize,\n            },\n          );\n        const mergedRect = mergePixelBboxesToRect([\n          adaptedResult.target,\n          ...(adaptedResult.references ?? []),\n        ]);\n        debugSection('mergedRect %j', mergedRect);\n        return { result, sectionError, mergedRect };\n      },\n      toParseError: (error, result) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        const message = result.content.error\n          ? `${result.content.error} (${parseErrorMessage})`\n          : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          result.contentString,\n          result.usage,\n          result.rawChoiceMessage,\n          result.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(msgs, modelRuntime, {\n        abortSignal: options.abortSignal,\n      }),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseXMLExtractionResponse<T>(rawResponse);\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInspect(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","assert","locateImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","locateRequest","userPromptToString","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","error","msg","Error","_elementDescription","modelRuntime","adapter","resultAdapter","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","preparedImage","prepareModelImage","imagePayload","msgs","callAiAndParseWithRetry","callAIWithObjectResponse","response","parseErrorMessage","modelErrors","message","AIResponseParseError","String","callError","errorMessage","AiLocateSection","sectionDescription","screenshotBase64","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","addOns","parsedResult","result","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","searchAreaConfig","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","callAI","reasoning_content","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC4DA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAC9B,MAAMC,eAAeD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAMO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvDa,IAAAA,sBAAAA,MAAAA,AAAAA,EACEF,0BACA;IAGF,MAAM,EAAEV,OAAO,EAAE,GAAGW;IACpB,MAAME,cAAcF,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaX,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMc,yBACJ,AAAoC,YAApC,OAAOJ,2BACHb,SACA,MAAMkB,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EACJC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6BN;IAErC,MAAMO,gBAAsC;QAC1C,wBAAwBC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBR;QAC3CG;QACAC;QACA,SAASH;IACX;IAEA,MAAMQ,gBAAgBpB,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMqB,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAC3BV,0BACAC,eACAM;IAEF,MAAM,EACJM,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,SAASA;YACT+B;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAMC,OAAOC,AAAAA,IAAAA,sCAAAA,eAAAA,AAAAA,EACXC,AAAAA,IAAAA,uCAAAA,yCAAAA,AAAAA,EACET,kBACAZ,cAAc,YAAY,EAAE;QAGhCvB,aAAa,WAAW0C;QAExB,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EACPH,MACAb,cAAc,sBAAsB;gBAEtC,QAAQ,EAAE;YACZ;YACA,GAAGY,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMrC;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ+B,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBgB,mBAAgC,EAChCtC,OAAsB,EACtBkB,aAAmC;IAEnC,MAAMqB,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAME,wBAAwBC,AAAAA,IAAAA,+BAAAA,iBAAAA,AAAAA,EAC5BzB,cAAc,sBAAsB;IAEtC,MAAM0B,eAAeC,AAAAA,IAAAA,+BAAAA,2BAAAA,AAAAA,EACnBL,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMM,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAa7B,cAAc,WAAW,CAAC,WAAW;QAClD,OAAOA,cAAc,WAAW,CAAC,KAAK;QACtC,QAAQA,cAAc,WAAW,CAAC,MAAM;QACxC,QAAQsB,QAAQ,eAAe;IACjC;IAEA,MAAMQ,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMN;gBACR;aACD;QACH;KACD;IAED,IAAIxB,cAAc,sBAAsB,EACtC+B,KAAK,IAAI,IAAI/B,cAAc,sBAAsB;IAGnD,IAAI;QACF,OAAO,MAAMgC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YACnC,QAAQ,IACNC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAkDF,MAAMV,cAAc;oBACpE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAYuC,aAAa,MAAM,CAAC,UAAU;oBAC1C,eAAeA,aAAa,MAAM,CAAC,aAAa;gBAClD;YACF,eAAe,CAACa;gBACd,MAAM3B,cAAc2B,SAAS,aAAa;gBAC1C,MAAMvB,SACJ,YAAYuB,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAG,EAAE;gBAC7D,IACE,CAAC5D,gBAAgB4D,SAAS,OAAO,EAAEX,cAAc,UAAU,CAAC,SAAS,GAErE,OAAO;oBACLhB;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;gBAGF,MAAML,mBACJiB,cAAc,mCAAmC,CAACW,SAAS,OAAO,EAAE;oBAClE,cAAcN,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBACF,OAAO;oBACLtB;oBACAC;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;YACF;YACA,cAAc,CAACM,OAAOiB;gBACpB,MAAMC,oBACJlB,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,MAAMmB,cACJ,YAAYF,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAGtD;gBAC3D,MAAMyD,UACJD,eAAeA,YAAY,MAAM,GAAG,IAChC,GAAGA,YAAY,IAAI,CAAC,MAAM,EAAE,EAAED,kBAAkB,CAAC,CAAC,GAClDA;gBACN,OAAO,IAAIG,yBAAAA,oBAAoBA,CAC7BD,SACAH,SAAS,aAAa,EACtBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBb,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAavC,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb9C,aACE,uDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,yBAAAA,oBAAoBA,EAC3C,OAAO;YACL,aAAaE,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,aAAaC;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF;AAEO,eAAeC,gBAAgB5D,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAE4D,kBAAkB,EAAE,GAAG7D;IACxC,MAAMuC,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAMsB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM6C,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAae;QACb,OAAO7D,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQuC,QAAQ,eAAe;IACjC;IAEA,MAAMI,eAAemB,AAAAA,IAAAA,uCAAAA,2BAAAA,AAAAA,EACnBvB,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAMwB,gCAAgCC,AAAAA,IAAAA,uCAAAA,yBAAAA,AAAAA,EACpC9C,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmB0C;IAErB,MAAMZ,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMkB;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOH,oBAAiC;QAC1C,MAAMK,SAAS,MAAMlD,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EACnBC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6B4C;QAE/BZ,KAAK,IAAI,IAAIiB;IACf;IAEA,IAAIC;IAYJ,IAAI;QACFA,eAAe,MAAMjB,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YAC3C,QAAQ,IACNC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAmDF,MAAMV,cAAc;oBACrE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAYuC,aAAa,MAAM,CAAC,UAAU;oBAC1C,eAAeA,aAAa,MAAM,CAAC,aAAa;gBAClD;YACF,eAAe,CAAC6B;gBACd,MAAMC,eAAeD,OAAO,OAAO,CAAC,KAAK;gBACzC,IACE,CAAC5E,gBAAgB4E,OAAO,OAAO,EAAE3B,cAAc,UAAU,CAAC,SAAS,GAEnE,OAAO;oBAAE2B;oBAAQC;gBAAa;gBAGhC,MAAMC,gBACJ7B,cAAc,wCAAwC,CACpD2B,OAAO,OAAO,EACd;oBACE,cAActB,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBAEJ,MAAMyB,aAAaC,AAAAA,IAAAA,sCAAAA,sBAAAA,AAAAA,EAAuB;oBACxCF,cAAc,MAAM;uBAChBA,cAAc,UAAU,IAAI,EAAE;iBACnC;gBACD/E,aAAa,iBAAiBgF;gBAC9B,OAAO;oBAAEH;oBAAQC;oBAAcE;gBAAW;YAC5C;YACA,cAAc,CAACpC,OAAOiC;gBACpB,MAAMf,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,MAAMoB,UAAUa,OAAO,OAAO,CAAC,KAAK,GAChC,GAAGA,OAAO,OAAO,CAAC,KAAK,CAAC,EAAE,EAAEf,kBAAkB,CAAC,CAAC,GAChDA;gBACJ,OAAO,IAAIG,yBAAAA,oBAAoBA,CAC7BD,SACAa,OAAO,aAAa,EACpBA,OAAO,KAAK,EACZA,OAAO,gBAAgB,EACvBA,OAAO,iBAAiB;YAE5B;YACA,iBAAiB7B,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAavC,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb5C,aACE,+DACA4C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,yBAAAA,oBAAoBA,EAC3C,OAAO;YACL,kBAAkB1D;YAClB,OAAO4D,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,kBAAkB5D;YAClB,OAAO,CAAC,eAAe,EAAE6D,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAES,MAAM,EAAEC,YAAY,EAAEE,UAAU,EAAE,GAAGJ;IAC7C,IAAI,CAACI,YACH,OAAO;QACL,kBAAkBzE;QAClB,OAAOuE;QACP,aAAaD,OAAO,aAAa;QACjC,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMK,eAAepE,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBkE,YAAYtE,QAAQ,QAAQ;QAClE,MAAMyE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1ClF,aAAa,2BAA2BkF;QAExC,MAAMG,mBAAmB,MAAM7E,sBAAsB;YACnDE;YACA,UAAUsE;QACZ;QAEAhF,aACE,uDACAmF,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOP;YACP,aAAaD,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOjC,OAAO;QACd,MAAMkB,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMwB,eAAeU,eACjB,GAAGA,aAAa,EAAE,EAAEhB,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBvD;YAClB,OAAO6D;YACP,aAAaS,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF;AAEO,eAAeS,qBAAwB7E,OAQ7C;IACC,MAAM,EAAE8E,SAAS,EAAE7E,OAAO,EAAE8E,aAAa,EAAEC,gBAAgB,EAAEzC,YAAY,EAAE,GACzEvC;IACF,MAAM4C,eAAeqC,AAAAA,IAAAA,8BAAAA,qBAAAA,AAAAA,EAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMlB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMiF,wBAAwBC,AAAAA,IAAAA,8BAAAA,sBAAAA,AAAAA,EAC5BnF,QAAQ,eAAe,IAAI,IAC3B8E;IAGF,MAAMM,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/C,MAAMM,qBAAqBpF,QAAQ,kBAAkB;QACrD,IAAIoF,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKtB;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAsB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMjC,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASwC;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAMd,SAAS,MAAMlD,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EAA+BgE;QACpD/B,KAAK,IAAI,IAAIiB;IACf;IAEA,OAAOhB,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,IACNsC,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAOvC,MAAMV,cAAc;gBACzB,aAAavC,QAAQ,WAAW;YAClC;QACF,eAAe,CAACoD;YACd,MAAM,EACJ,SAAS3B,WAAW,EACpBE,KAAK,EACL8D,iBAAiB,EACjB/D,gBAAgB,EACjB,GAAG0B;YACJ,MAAMsC,cAAcC,AAAAA,IAAAA,8BAAAA,0BAAAA,AAAAA,EAA8BlE;YAClD,OAAO;gBACLiE;gBACAjE;gBACAC;gBACAC;gBACA8D;YACF;QACF;QACA,cAAc,CAACG,YAAYxC;YACzB,MAAMO,eACJiC,sBAAsBvD,QAAQuD,WAAW,OAAO,GAAGnC,OAAOmC;YAC5D,OAAO,IAAIpC,yBAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEG,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBb,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAavC,QAAQ,WAAW;QAChC,cAAc,CAACmC;YACb9C,aACE,iDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;QAEpD;IACF;AACF;AAEO,eAAe0D,sBACpBC,WAAmB,EACnBvD,YAA0B;IAK1B,MAAMK,eAAemD,AAAAA,IAAAA,yCAAAA,iCAAAA,AAAAA;IACrB,MAAMC,aAAaC,AAAAA,IAAAA,yCAAAA,yBAAAA,AAAAA,EAA0BH;IAE7C,MAAM7C,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASoD;QACX;KACD;IAED3G,aAAa,yCAAyCyG;IAEtD,MAAM1B,SAAS,MAAMjB,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACnBF,MACAV,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB6B,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/inspect.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequestContext,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\ntype SectionLocateObjectResponse = Awaited<\n  ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n>;\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const referenceImageMessages =\n    typeof targetElementDescription === 'string'\n      ? undefined\n      : await multimodalPromptToChatMessages(\n          userPromptToMultimodalPrompt(targetElementDescription),\n        );\n  const locateRequest: LocateRequestContext = {\n    elementDescriptionText: userPromptToString(targetElementDescription),\n    locateImage,\n    referenceImageMessages,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(\n    targetElementDescription,\n    locateOptions,\n    locateRequest,\n  );\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugInspect('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          locateRequest.elementDescriptionText,\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  _elementDescription: TUserPrompt,\n  options: LocateOptions,\n  locateRequest: LocateRequestContext,\n): Promise<LocateModelResponse> {\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const userInstructionPrompt = findElementPrompt(\n    locateRequest.elementDescriptionText,\n  );\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateRequest.locateImage.imageBase64,\n    width: locateRequest.locateImage.width,\n    height: locateRequest.locateImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (locateRequest.referenceImageMessages) {\n    msgs.push(...locateRequest.referenceImageMessages);\n  }\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AIElementLocateResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'locate',\n          retryTimes: modelRuntime.config.retryCount,\n          retryInterval: modelRuntime.config.retryInterval,\n        }),\n      parseResponse: (response): LocateModelResponse => {\n        const rawResponse = response.contentString;\n        const errors: string[] | undefined =\n          'errors' in response.content ? response.content.errors : [];\n        if (\n          !hasLocateResult(response.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: errors as string[],\n          };\n        }\n\n        const locatedPixelBbox =\n          resultAdapter.adaptElementLocateResultToPixelBbox(response.content, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n        return {\n          locatedPixelBbox,\n          rawResponse,\n          rawChoiceMessage: response.rawChoiceMessage,\n          usage: response.usage,\n          reasoningContent: response.reasoning_content,\n          errors: errors as string[],\n        };\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        const modelErrors =\n          'errors' in response.content ? response.content.errors : undefined;\n        const message =\n          modelErrors && modelErrors.length > 0\n            ? `${modelErrors.join('\\n')} (${parseErrorMessage})`\n            : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          response.contentString,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugInspect(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let parsedResult:\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AISectionLocatorResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'section-locator',\n          retryTimes: modelRuntime.config.retryCount,\n          retryInterval: modelRuntime.config.retryInterval,\n        }),\n      parseResponse: (result) => {\n        const sectionError = result.content.error;\n        if (\n          !hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return { result, sectionError };\n        }\n\n        const adaptedResult =\n          resultAdapter.adaptSectionLocateResultToPixelBboxGroup(\n            result.content,\n            {\n              preparedSize: preparedImage.preparedSize,\n              contentSize: preparedImage.contentSize,\n            },\n          );\n        const mergedRect = mergePixelBboxesToRect([\n          adaptedResult.target,\n          ...(adaptedResult.references ?? []),\n        ]);\n        debugSection('mergedRect %j', mergedRect);\n        return { result, sectionError, mergedRect };\n      },\n      toParseError: (error, result) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        const message = result.content.error\n          ? `${result.content.error} (${parseErrorMessage})`\n          : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          result.contentString,\n          result.usage,\n          result.rawChoiceMessage,\n          result.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    extractOption?.context,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(msgs, modelRuntime, {\n        abortSignal: options.abortSignal,\n      }),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseXMLExtractionResponse<T>(rawResponse);\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInspect(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","assert","locateImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","locateRequest","userPromptToString","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","error","msg","Error","_elementDescription","modelRuntime","adapter","resultAdapter","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","preparedImage","prepareModelImage","imagePayload","msgs","callAiAndParseWithRetry","callAIWithObjectResponse","response","parseErrorMessage","modelErrors","message","AIResponseParseError","String","callError","errorMessage","AiLocateSection","sectionDescription","screenshotBase64","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","addOns","parsedResult","result","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","searchAreaConfig","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","callAI","reasoning_content","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC4DA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAC9B,MAAMC,eAAeD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAMO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvDa,IAAAA,sBAAAA,MAAAA,AAAAA,EACEF,0BACA;IAGF,MAAM,EAAEV,OAAO,EAAE,GAAGW;IACpB,MAAME,cAAcF,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaX,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMc,yBACJ,AAAoC,YAApC,OAAOJ,2BACHb,SACA,MAAMkB,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EACJC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6BN;IAErC,MAAMO,gBAAsC;QAC1C,wBAAwBC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBR;QAC3CG;QACAC;QACA,SAASH;IACX;IAEA,MAAMQ,gBAAgBpB,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMqB,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAC3BV,0BACAC,eACAM;IAEF,MAAM,EACJM,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,SAASA;YACT+B;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAMC,OAAOC,AAAAA,IAAAA,sCAAAA,eAAAA,AAAAA,EACXC,AAAAA,IAAAA,uCAAAA,yCAAAA,AAAAA,EACET,kBACAZ,cAAc,YAAY,EAAE;QAGhCvB,aAAa,WAAW0C;QAExB,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EACPH,MACAb,cAAc,sBAAsB;gBAEtC,QAAQ,EAAE;YACZ;YACA,GAAGY,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMrC;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ+B,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBgB,mBAAgC,EAChCtC,OAAsB,EACtBkB,aAAmC;IAEnC,MAAMqB,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAME,wBAAwBC,AAAAA,IAAAA,+BAAAA,iBAAAA,AAAAA,EAC5BzB,cAAc,sBAAsB;IAEtC,MAAM0B,eAAeC,AAAAA,IAAAA,+BAAAA,2BAAAA,AAAAA,EACnBL,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMM,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAa7B,cAAc,WAAW,CAAC,WAAW;QAClD,OAAOA,cAAc,WAAW,CAAC,KAAK;QACtC,QAAQA,cAAc,WAAW,CAAC,MAAM;QACxC,QAAQsB,QAAQ,eAAe;IACjC;IAEA,MAAMQ,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMN;gBACR;aACD;QACH;KACD;IAED,IAAIxB,cAAc,sBAAsB,EACtC+B,KAAK,IAAI,IAAI/B,cAAc,sBAAsB;IAGnD,IAAI;QACF,OAAO,MAAMgC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YACnC,QAAQ,IACNC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAkDF,MAAMV,cAAc;oBACpE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAYuC,aAAa,MAAM,CAAC,UAAU;oBAC1C,eAAeA,aAAa,MAAM,CAAC,aAAa;gBAClD;YACF,eAAe,CAACa;gBACd,MAAM3B,cAAc2B,SAAS,aAAa;gBAC1C,MAAMvB,SACJ,YAAYuB,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAG,EAAE;gBAC7D,IACE,CAAC5D,gBAAgB4D,SAAS,OAAO,EAAEX,cAAc,UAAU,CAAC,SAAS,GAErE,OAAO;oBACLhB;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;gBAGF,MAAML,mBACJiB,cAAc,mCAAmC,CAACW,SAAS,OAAO,EAAE;oBAClE,cAAcN,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBACF,OAAO;oBACLtB;oBACAC;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;YACF;YACA,cAAc,CAACM,OAAOiB;gBACpB,MAAMC,oBACJlB,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,MAAMmB,cACJ,YAAYF,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAGtD;gBAC3D,MAAMyD,UACJD,eAAeA,YAAY,MAAM,GAAG,IAChC,GAAGA,YAAY,IAAI,CAAC,MAAM,EAAE,EAAED,kBAAkB,CAAC,CAAC,GAClDA;gBACN,OAAO,IAAIG,yBAAAA,oBAAoBA,CAC7BD,SACAH,SAAS,aAAa,EACtBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBb,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAavC,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb9C,aACE,uDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,yBAAAA,oBAAoBA,EAC3C,OAAO;YACL,aAAaE,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,aAAaC;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF;AAEO,eAAeC,gBAAgB5D,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAE4D,kBAAkB,EAAE,GAAG7D;IACxC,MAAMuC,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAMsB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM6C,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAae;QACb,OAAO7D,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQuC,QAAQ,eAAe;IACjC;IAEA,MAAMI,eAAemB,AAAAA,IAAAA,uCAAAA,2BAAAA,AAAAA,EACnBvB,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAMwB,gCAAgCC,AAAAA,IAAAA,uCAAAA,yBAAAA,AAAAA,EACpC9C,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmB0C;IAErB,MAAMZ,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMkB;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOH,oBAAiC;QAC1C,MAAMK,SAAS,MAAMlD,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EACnBC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6B4C;QAE/BZ,KAAK,IAAI,IAAIiB;IACf;IAEA,IAAIC;IAYJ,IAAI;QACFA,eAAe,MAAMjB,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YAC3C,QAAQ,IACNC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAmDF,MAAMV,cAAc;oBACrE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAYuC,aAAa,MAAM,CAAC,UAAU;oBAC1C,eAAeA,aAAa,MAAM,CAAC,aAAa;gBAClD;YACF,eAAe,CAAC6B;gBACd,MAAMC,eAAeD,OAAO,OAAO,CAAC,KAAK;gBACzC,IACE,CAAC5E,gBAAgB4E,OAAO,OAAO,EAAE3B,cAAc,UAAU,CAAC,SAAS,GAEnE,OAAO;oBAAE2B;oBAAQC;gBAAa;gBAGhC,MAAMC,gBACJ7B,cAAc,wCAAwC,CACpD2B,OAAO,OAAO,EACd;oBACE,cAActB,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBAEJ,MAAMyB,aAAaC,AAAAA,IAAAA,sCAAAA,sBAAAA,AAAAA,EAAuB;oBACxCF,cAAc,MAAM;uBAChBA,cAAc,UAAU,IAAI,EAAE;iBACnC;gBACD/E,aAAa,iBAAiBgF;gBAC9B,OAAO;oBAAEH;oBAAQC;oBAAcE;gBAAW;YAC5C;YACA,cAAc,CAACpC,OAAOiC;gBACpB,MAAMf,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,MAAMoB,UAAUa,OAAO,OAAO,CAAC,KAAK,GAChC,GAAGA,OAAO,OAAO,CAAC,KAAK,CAAC,EAAE,EAAEf,kBAAkB,CAAC,CAAC,GAChDA;gBACJ,OAAO,IAAIG,yBAAAA,oBAAoBA,CAC7BD,SACAa,OAAO,aAAa,EACpBA,OAAO,KAAK,EACZA,OAAO,gBAAgB,EACvBA,OAAO,iBAAiB;YAE5B;YACA,iBAAiB7B,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAavC,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb5C,aACE,+DACA4C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,yBAAAA,oBAAoBA,EAC3C,OAAO;YACL,kBAAkB1D;YAClB,OAAO4D,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,kBAAkB5D;YAClB,OAAO,CAAC,eAAe,EAAE6D,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAES,MAAM,EAAEC,YAAY,EAAEE,UAAU,EAAE,GAAGJ;IAC7C,IAAI,CAACI,YACH,OAAO;QACL,kBAAkBzE;QAClB,OAAOuE;QACP,aAAaD,OAAO,aAAa;QACjC,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMK,eAAepE,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBkE,YAAYtE,QAAQ,QAAQ;QAClE,MAAMyE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1ClF,aAAa,2BAA2BkF;QAExC,MAAMG,mBAAmB,MAAM7E,sBAAsB;YACnDE;YACA,UAAUsE;QACZ;QAEAhF,aACE,uDACAmF,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOP;YACP,aAAaD,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOjC,OAAO;QACd,MAAMkB,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMwB,eAAeU,eACjB,GAAGA,aAAa,EAAE,EAAEhB,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBvD;YAClB,OAAO6D;YACP,aAAaS,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF;AAEO,eAAeS,qBAAwB7E,OAQ7C;IACC,MAAM,EAAE8E,SAAS,EAAE7E,OAAO,EAAE8E,aAAa,EAAEC,gBAAgB,EAAEzC,YAAY,EAAE,GACzEvC;IACF,MAAM4C,eAAeqC,AAAAA,IAAAA,8BAAAA,qBAAAA,AAAAA,EAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMlB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMiF,wBAAwBC,AAAAA,IAAAA,8BAAAA,sBAAAA,AAAAA,EAC5BnF,QAAQ,eAAe,IAAI,IAC3B8E,WACAC,eAAe;IAGjB,MAAMK,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/C,MAAMM,qBAAqBpF,QAAQ,kBAAkB;QACrD,IAAIoF,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKtB;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAsB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMjC,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASwC;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAMd,SAAS,MAAMlD,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EAA+BgE;QACpD/B,KAAK,IAAI,IAAIiB;IACf;IAEA,OAAOhB,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,IACNsC,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAOvC,MAAMV,cAAc;gBACzB,aAAavC,QAAQ,WAAW;YAClC;QACF,eAAe,CAACoD;YACd,MAAM,EACJ,SAAS3B,WAAW,EACpBE,KAAK,EACL8D,iBAAiB,EACjB/D,gBAAgB,EACjB,GAAG0B;YACJ,MAAMsC,cAAcC,AAAAA,IAAAA,8BAAAA,0BAAAA,AAAAA,EAA8BlE;YAClD,OAAO;gBACLiE;gBACAjE;gBACAC;gBACAC;gBACA8D;YACF;QACF;QACA,cAAc,CAACG,YAAYxC;YACzB,MAAMO,eACJiC,sBAAsBvD,QAAQuD,WAAW,OAAO,GAAGnC,OAAOmC;YAC5D,OAAO,IAAIpC,yBAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEG,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBb,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAavC,QAAQ,WAAW;QAChC,cAAc,CAACmC;YACb9C,aACE,iDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;QAEpD;IACF;AACF;AAEO,eAAe0D,sBACpBC,WAAmB,EACnBvD,YAA0B;IAK1B,MAAMK,eAAemD,AAAAA,IAAAA,yCAAAA,iCAAAA,AAAAA;IACrB,MAAMC,aAAaC,AAAAA,IAAAA,yCAAAA,yBAAAA,AAAAA,EAA0BH;IAE7C,MAAM7C,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASoD;QACX;KACD;IAED3G,aAAa,yCAAyCyG;IAEtD,MAAM1B,SAAS,MAAMjB,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EACnBF,MACAV,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB6B,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/lib/ai-model/llm-planning.js

@@ -274,7 +274,8 @@ async function plan(userInstruction, opts) {
         if (planFromAI.log) conversationHistory.appendSubGoalLog(planFromAI.log);
     } else if (planFromAI.log) conversationHistory.appendHistoricalLog(planFromAI.log);
     if (planFromAI.memory) conversationHistory.appendMemory(planFromAI.memory);
-    conversationHistory.append({
+    if (modelRuntime.adapter.chatCompletion.replayRawAssistantMessage && rawChoiceMessage) conversationHistory.append(rawChoiceMessage);
+    else conversationHistory.append({
         role: 'assistant',
         content: [
             {

package/dist/lib/ai-model/llm-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type TUserPrompt, userPromptToString } from '@/common';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../common';\nimport { planningModelFamilyRequiredForLocateMessage } from './errors';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport { AIResponseParseError, callAI } from './service-caller/index';\nimport type { JsonParser, JsonParserSource } from './service-caller/json';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport type {\n  LocateResultAdapter,\n  LocateResultContext,\n} from './shared/model-locate-result';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport { normalizePlanningActionLocateFields } from './workflows/planning/locate-normalization';\nimport type { PlanOptions } from './workflows/planning/types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse.\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  jsonParser: JsonParser,\n): RawResponsePlanningAIResponse {\n  // Use <planning> instead of <thought> to avoid colliding with Gemini thought\n  // summaries, which may also be emitted as <thought> in OpenAI-compatible\n  // response content.\n  const thought = extractXMLTag(xmlString, 'planning');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    // Strip any trailing XML tags that LLM might have leaked into the action type\n    // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultAdapter?: LocateResultAdapter;\n  locateResultContext: LocateResultContext;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultAdapter,\n    locateResultContext,\n  } = options;\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(messages, modelRuntime, {\n        abortSignal,\n        requiresOriginalImageDetail: includeLocateInPlanning,\n      }),\n    parseResponse: (response) => {\n      const planFromAI = parseXMLPlanningResponse(\n        response.content,\n        modelRuntime.adapter.jsonParser,\n      );\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      // Keep yamlFlow based on the model's original action params. Coordinate\n      // normalization adds runtime-only locatedPixelBbox fields afterwards.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultAdapter,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function plan(\n  userInstruction: TUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultAdapter =\n    modelRuntime.config.modelFamily && adapter.locate.kind === 'standard'\n      ? adapter.locate.resultAdapter\n      : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    locatePromptSpec: locateResultAdapter?.promptSpec,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const userInstructionText = userPromptToString(userInstruction);\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = opts.referenceImageMessages ?? [];\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstructionText}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultAdapter,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: [\n      {\n        type: 'text',\n        text: rawResponse,\n      },\n    ],\n  });\n\n  return returnValue;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","noPreviousActionsText","parseXMLPlanningResponse","xmlString","jsonParser","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","e","Error","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultAdapter","locateResultContext","callAiAndParseWithRetry","callAI","response","planFromAI","actions","yamlFlow","buildYamlFlowFromPlans","normalizePlanningActionLocateFields","parseError","errorMessage","String","AIResponseParseError","plan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","preparedImage","prepareModelImage","imagePayload","userInstructionText","userPromptToString","actionContext","referenceImageMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","assert","index"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;ACsBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAKK,SAASC,yBACdC,SAAiB,EACjBC,UAAsB;IAKtB,MAAMC,UAAUC,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACzC,MAAMI,SAASD,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACxC,MAAMK,MAAMF,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACvC,MAAMO,aAAaJ,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,AAAAA,IAAAA,wBAAAA,oBAAAA,AAAAA,EAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,AAAAA,IAAAA,wBAAAA,wBAAAA,AAAAA,EAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQrB,WAAWO,gBAAgB;gBACjC,QAAQ;gBACR,yBACEa,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGR;YACjD;QACF,EAAE,OAAOU,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFH,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAcA,eAAeO,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,mBAAmB,EACnBC,mBAAmB,EACpB,GAAGP;IACJ,OAAOQ,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,IACNC,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAOR,UAAUC,cAAc;gBAC7BC;gBACA,6BAA6BC;YAC/B;QACF,eAAe,CAACM;YACd,MAAMC,aAAatC,yBACjBqC,SAAS,OAAO,EAChBR,aAAa,OAAO,CAAC,UAAU;YAEjC,IAAIS,WAAW,MAAM,IAAIA,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;gBACjExC,QACE;gBAEFwC,WAAW,eAAe,GAAGxB;gBAC7BwB,WAAW,eAAe,GAAGxB;YAC/B;YAEA,MAAMyB,UAAUD,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAG5D,MAAME,WAAWC,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBF,SAASP;YACjDU,IAAAA,wCAAAA,mCAAAA,AAAAA,EAAoCH,SAAS;gBAC3CP;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEG;gBAAUC;gBAAYC;gBAASC;YAAS;QACnD;QACA,cAAc,CAACG,YAAYN;YACzB,MAAMO,eACJD,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,yBAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEF,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBR,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACa;YACb/C,MACE,mDACA+C,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,KACpBC,eAA4B,EAC5BC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAMpB,eAAeoB,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGvB;IACpB,MAAM,EAAEwB,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,IAAID,KAAK,uBAAuB,IAAI,CAACpB,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIJ,MACR8B,AAAAA,IAAAA,mCAAAA,2CAAAA,AAAAA,EAA4C1B,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,sBACJJ,aAAa,MAAM,CAAC,WAAW,IAAIuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,GAClDA,QAAQ,MAAM,CAAC,aAAa,GAC5BtC;IAGN,MAAM0C,kBAAkBP,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMQ,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAaT,KAAK,WAAW;QAC7B,kBAAkBhB,qBAAqB;QACvC,yBAAyBgB,KAAK,uBAAuB;QACrD,gBAAgB;QAChBO;IACF;IAEA,MAAMG,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaN;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMS,eAAeF,cAAc,WAAW;IAE9C,MAAMG,sBAAsBC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBf;IAC/C,MAAMgB,gBAAgBf,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMgB,yBAAyBhB,KAAK,sBAAsB,IAAI,EAAE;IAChE,MAAMiB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGF,cAAc,kBAAkB,EAAEF,oBAAoB,mBAAmB,CAAC;gBACrF;aACD;QACH;WACGG;KACJ;IAED,IAAIE;IAKJ,MAAMC,wBAAwBZ,kBAC1BL,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpD,uBAAuB;IAGpC,MAAMuE,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAV,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFV,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACDvC,UAAU,EACVC,OAAO,EACPC,QAAQ,EACT,GAAG,MAAMd,6BAA6B;QACrC,UAAU+C;QACV5C;QACA,aAAaoB,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BhB;QACA,qBAAqB;YACnB,cAAc0B,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;IACF;IAEA,IAAImB,yBAAyB;IAG7B,IAAIxC,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;QAC5C1C,MAAM;QACNkF,yBAAyB;QAEzB,IAAItB,iBACFL,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAGzC,UAAU;QACbC;QACAmC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO1C,YAAY;IAGnB,IAAIkB,iBAAiB;QACnB,IAAIlB,WAAW,cAAc,EAAE,QAC7Ba,oBAAoB,aAAa,CAACb,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM2C,SAAS3C,WAAW,mBAAmB,CAChDa,oBAAoB,mBAAmB,CAAC8B;QAI5C,IAAI3C,WAAW,GAAG,EAChBa,oBAAoB,gBAAgB,CAACb,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBa,oBAAoB,mBAAmB,CAACb,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBa,oBAAoB,YAAY,CAACb,WAAW,MAAM;IAGpDa,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAEA,OAAOK;AACT"}
+{"version":3,"file":"ai-model/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type TUserPrompt, userPromptToString } from '@/common';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../common';\nimport { planningModelFamilyRequiredForLocateMessage } from './errors';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n  extractXMLTag,\n  parseMarkFinishedIndexes,\n  parseSubGoalsFromXML,\n} from './prompt/util';\nimport { AIResponseParseError, callAI } from './service-caller/index';\nimport type { JsonParser, JsonParserSource } from './service-caller/json';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport type {\n  LocateResultAdapter,\n  LocateResultContext,\n} from './shared/model-locate-result';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport { normalizePlanningActionLocateFields } from './workflows/planning/locate-normalization';\nimport type { PlanOptions } from './workflows/planning/types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\n\n/**\n * Parse XML response from LLM and convert to RawResponsePlanningAIResponse.\n */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  jsonParser: JsonParser,\n): RawResponsePlanningAIResponse {\n  // Use <planning> instead of <thought> to avoid colliding with Gemini thought\n  // summaries, which may also be emitted as <thought> in OpenAI-compatible\n  // response content.\n  const thought = extractXMLTag(xmlString, 'planning');\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n  const actionType = extractXMLTag(xmlString, 'action-type');\n  const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n  // Parse <complete> tag with success attribute\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  // Parse sub-goal related tags\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  // Parse action\n  let action: any = null;\n  if (actionType && actionType.toLowerCase() !== 'null') {\n    // Strip any trailing XML tags that LLM might have leaked into the action type\n    // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        // Parse the JSON string in action-param-json\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n      } catch (e) {\n        throw new Error(`Failed to parse action-param-json: ${e}`);\n      }\n    }\n\n    action = {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    ...(memory ? { memory } : {}),\n    log,\n    ...(error ? { error } : {}),\n    action,\n    ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n    ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n    ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n    ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n  };\n}\n\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultAdapter?: LocateResultAdapter;\n  locateResultContext: LocateResultContext;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultAdapter,\n    locateResultContext,\n  } = options;\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(messages, modelRuntime, {\n        abortSignal,\n        requiresOriginalImageDetail: includeLocateInPlanning,\n      }),\n    parseResponse: (response) => {\n      const planFromAI = parseXMLPlanningResponse(\n        response.content,\n        modelRuntime.adapter.jsonParser,\n      );\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      // Keep yamlFlow based on the model's original action params. Coordinate\n      // normalization adds runtime-only locatedPixelBbox fields afterwards.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultAdapter,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function plan(\n  userInstruction: TUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultAdapter =\n    modelRuntime.config.modelFamily && adapter.locate.kind === 'standard'\n      ? adapter.locate.resultAdapter\n      : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.deepThink === true;\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    locatePromptSpec: locateResultAdapter?.promptSpec,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    includeThought: true, // always include thought\n    includeSubGoals,\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const userInstructionText = userPromptToString(userInstruction);\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = opts.referenceImageMessages ?? [];\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstructionText}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultAdapter,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  // Some model providers require opaque assistant fields to be replayed\n  // verbatim in later turns. Keep this opt-in per model adapter so that an\n  // unverified provider does not receive non-standard response fields.\n  if (\n    modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n    rawChoiceMessage\n  ) {\n    conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n  } else {\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","noPreviousActionsText","parseXMLPlanningResponse","xmlString","jsonParser","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","e","Error","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultAdapter","locateResultContext","callAiAndParseWithRetry","callAI","response","planFromAI","actions","yamlFlow","buildYamlFlowFromPlans","normalizePlanningActionLocateFields","parseError","errorMessage","String","AIResponseParseError","plan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","preparedImage","prepareModelImage","imagePayload","userInstructionText","userPromptToString","actionContext","referenceImageMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","assert","index"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;ACsBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAKK,SAASC,yBACdC,SAAiB,EACjBC,UAAsB;IAKtB,MAAMC,UAAUC,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACzC,MAAMI,SAASD,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACxC,MAAMK,MAAMF,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACvC,MAAMO,aAAaJ,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,AAAAA,IAAAA,wBAAAA,aAAAA,AAAAA,EAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,AAAAA,IAAAA,wBAAAA,oBAAAA,AAAAA,EAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,AAAAA,IAAAA,wBAAAA,wBAAAA,AAAAA,EAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQrB,WAAWO,gBAAgB;gBACjC,QAAQ;gBACR,yBACEa,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAGR;YACjD;QACF,EAAE,OAAOU,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFH,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAcA,eAAeO,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,mBAAmB,EACnBC,mBAAmB,EACpB,GAAGP;IACJ,OAAOQ,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,IACNC,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAOR,UAAUC,cAAc;gBAC7BC;gBACA,6BAA6BC;YAC/B;QACF,eAAe,CAACM;YACd,MAAMC,aAAatC,yBACjBqC,SAAS,OAAO,EAChBR,aAAa,OAAO,CAAC,UAAU;YAEjC,IAAIS,WAAW,MAAM,IAAIA,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;gBACjExC,QACE;gBAEFwC,WAAW,eAAe,GAAGxB;gBAC7BwB,WAAW,eAAe,GAAGxB;YAC/B;YAEA,MAAMyB,UAAUD,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAG5D,MAAME,WAAWC,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBF,SAASP;YACjDU,IAAAA,wCAAAA,mCAAAA,AAAAA,EAAoCH,SAAS;gBAC3CP;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEG;gBAAUC;gBAAYC;gBAASC;YAAS;QACnD;QACA,cAAc,CAACG,YAAYN;YACzB,MAAMO,eACJD,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,yBAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEF,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBR,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACa;YACb/C,MACE,mDACA+C,sBAAsBlB,QAAQkB,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,KACpBC,eAA4B,EAC5BC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAMpB,eAAeoB,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGvB;IACpB,MAAM,EAAEwB,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,IAAID,KAAK,uBAAuB,IAAI,CAACpB,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIJ,MACR8B,AAAAA,IAAAA,mCAAAA,2CAAAA,AAAAA,EAA4C1B,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,sBACJJ,aAAa,MAAM,CAAC,WAAW,IAAIuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,GAClDA,QAAQ,MAAM,CAAC,aAAa,GAC5BtC;IAGN,MAAM0C,kBAAkBP,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMQ,eAAe,MAAMC,AAAAA,IAAAA,gCAAAA,0BAAAA,AAAAA,EAA2B;QACpD,aAAaT,KAAK,WAAW;QAC7B,kBAAkBhB,qBAAqB;QACvC,yBAAyBgB,KAAK,uBAAuB;QACrD,gBAAgB;QAChBO;IACF;IAEA,MAAMG,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaN;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMS,eAAeF,cAAc,WAAW;IAE9C,MAAMG,sBAAsBC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBf;IAC/C,MAAMgB,gBAAgBf,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMgB,yBAAyBhB,KAAK,sBAAsB,IAAI,EAAE;IAChE,MAAMiB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGF,cAAc,kBAAkB,EAAEF,oBAAoB,mBAAmB,CAAC;gBACrF;aACD;QACH;WACGG;KACJ;IAED,IAAIE;IAKJ,MAAMC,wBAAwBZ,kBAC1BL,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpD,uBAAuB;IAGpC,MAAMuE,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAV,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKR;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFV,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACDvC,UAAU,EACVC,OAAO,EACPC,QAAQ,EACT,GAAG,MAAMd,6BAA6B;QACrC,UAAU+C;QACV5C;QACA,aAAaoB,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BhB;QACA,qBAAqB;YACnB,cAAc0B,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;IACF;IAEA,IAAImB,yBAAyB;IAG7B,IAAIxC,AAA+BxB,WAA/BwB,WAAW,eAAe,EAAgB;QAC5C1C,MAAM;QACNkF,yBAAyB;QAEzB,IAAItB,iBACFL,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAGzC,UAAU;QACbC;QACAmC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO1C,YAAY;IAGnB,IAAIkB,iBAAiB;QACnB,IAAIlB,WAAW,cAAc,EAAE,QAC7Ba,oBAAoB,aAAa,CAACb,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM2C,SAAS3C,WAAW,mBAAmB,CAChDa,oBAAoB,mBAAmB,CAAC8B;QAI5C,IAAI3C,WAAW,GAAG,EAChBa,oBAAoB,gBAAgB,CAACb,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBa,oBAAoB,mBAAmB,CAACb,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBa,oBAAoB,YAAY,CAACb,WAAW,MAAM;IAMpD,IACET,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7DgD,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOK;AACT"}

package/dist/lib/ai-model/model-adapter/chat-completion.js

@@ -63,6 +63,7 @@ function resolveChatCompletion(chatCompletion) {
     const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];
     const extractContentAndReasoning = resolveExtractContentAndReasoning(chatCompletion);
     const useReasoningAsContentFallback = chatCompletion?.useReasoningAsContentFallback ?? false;
+    const replayRawAssistantMessage = chatCompletion?.replayRawAssistantMessage ?? false;
     return {
         unsupportedUserConfig,
         buildChatCompletionParams: (input)=>{
@@ -79,7 +80,8 @@ function resolveChatCompletion(chatCompletion) {
                 midsceneDefaults: midsceneChatCompletionDefaults
             }),
         extractContentAndReasoning,
-        useReasoningAsContentFallback
+        useReasoningAsContentFallback,
+        replayRawAssistantMessage
     };
 }
 exports.resolveChatCompletion = __webpack_exports__.resolveChatCompletion;

package/dist/lib/ai-model/model-adapter/chat-completion.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/chat-completion.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/model-adapter/chat-completion.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  ChatCompletionAdapter,\n  ChatCompletionCallContext,\n  ExtractContentAndReasoning,\n  MidsceneChatCompletionDefaults,\n  ModelAdapterDefinition,\n} from './types';\n\nconst defaultImageDetail = (_input: unknown) => undefined;\n\nconst defaultChatCompletionParams = ({\n  midsceneDefaults,\n  userConfig,\n}: ChatCompletionCallContext) => ({\n  config: {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n  },\n});\n\nconst midsceneChatCompletionDefaults: MidsceneChatCompletionDefaults = {\n  temperature: 0,\n};\n\nfunction createDefaultExtractContentAndReasoning(\n  reasoningContentKeys: string[],\n): ExtractContentAndReasoning {\n  return (message) => {\n    if (!message) {\n      return {\n        content: '',\n        reasoning_content: '',\n      };\n    }\n\n    const messageRecord = message as Record<string, unknown>;\n    const rawReasoning = reasoningContentKeys\n      .map((key) => messageRecord[key])\n      .find((value) => typeof value === 'string');\n    const messageReasoning =\n      typeof rawReasoning === 'string' ? rawReasoning : '';\n\n    return {\n      content: typeof message.content === 'string' ? message.content : '',\n      reasoning_content: messageReasoning,\n    };\n  };\n}\n\nfunction resolveExtractContentAndReasoning(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ExtractContentAndReasoning {\n  const messageExtraction = chatCompletion?.messageExtraction;\n  if (messageExtraction?.kind === 'custom') {\n    return messageExtraction.extractContentAndReasoning;\n  }\n\n  return createDefaultExtractContentAndReasoning(\n    messageExtraction?.reasoningContentKeys ?? ['reasoning_content'],\n  );\n}\n\nexport function resolveChatCompletion(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ChatCompletionAdapter {\n  const buildChatCompletionParams =\n    chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;\n  const resolveImageDetail =\n    chatCompletion?.resolveImageDetail ?? defaultImageDetail;\n  const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];\n  const extractContentAndReasoning =\n    resolveExtractContentAndReasoning(chatCompletion);\n  const useReasoningAsContentFallback =\n    chatCompletion?.useReasoningAsContentFallback ?? false;\n\n  return {\n    unsupportedUserConfig,\n    buildChatCompletionParams: (input) => {\n      const context = {\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      };\n      return buildChatCompletionParams(context);\n    },\n    resolveImageDetail: (input) =>\n      resolveImageDetail({\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      }),\n    extractContentAndReasoning,\n    useReasoningAsContentFallback,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultImageDetail","_input","undefined","defaultChatCompletionParams","midsceneDefaults","userConfig","midsceneChatCompletionDefaults","createDefaultExtractContentAndReasoning","reasoningContentKeys","message","messageRecord","rawReasoning","value","messageReasoning","resolveExtractContentAndReasoning","chatCompletion","messageExtraction","resolveChatCompletion","buildChatCompletionParams","resolveImageDetail","unsupportedUserConfig","extractContentAndReasoning","useReasoningAsContentFallback","input","context"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;ACEA,MAAMI,qBAAqB,CAACC,SAAoBC;AAEhD,MAAMC,8BAA8B,CAAC,EACnCC,gBAAgB,EAChBC,UAAU,EACgB,GAAM;QAChC,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEA,MAAME,iCAAiE;IACrE,aAAa;AACf;AAEA,SAASC,wCACPC,oBAA8B;IAE9B,OAAO,CAACC;QACN,IAAI,CAACA,SACH,OAAO;YACL,SAAS;YACT,mBAAmB;QACrB;QAGF,MAAMC,gBAAgBD;QACtB,MAAME,eAAeH,qBAClB,GAAG,CAAC,CAACb,MAAQe,aAAa,CAACf,IAAI,EAC/B,IAAI,CAAC,CAACiB,QAAU,AAAiB,YAAjB,OAAOA;QAC1B,MAAMC,mBACJ,AAAwB,YAAxB,OAAOF,eAA4BA,eAAe;QAEpD,OAAO;YACL,SAAS,AAA2B,YAA3B,OAAOF,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;YACjE,mBAAmBI;QACrB;IACF;AACF;AAEA,SAASC,kCACPC,cAAwD;IAExD,MAAMC,oBAAoBD,gBAAgB;IAC1C,IAAIC,mBAAmB,SAAS,UAC9B,OAAOA,kBAAkB,0BAA0B;IAGrD,OAAOT,wCACLS,mBAAmB,wBAAwB;QAAC;KAAoB;AAEpE;AAEO,SAASC,sBACdF,cAAwD;IAExD,MAAMG,4BACJH,gBAAgB,6BAA6BZ;IAC/C,MAAMgB,qBACJJ,gBAAgB,sBAAsBf;IACxC,MAAMoB,wBAAwBL,gBAAgB,yBAAyB,EAAE;IACzE,MAAMM,6BACJP,kCAAkCC;IACpC,MAAMO,gCACJP,gBAAgB,iCAAiC;IAEnD,OAAO;QACLK;QACA,2BAA2B,CAACG;YAC1B,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBjB;YACpB;YACA,OAAOY,0BAA0BM;QACnC;QACA,oBAAoB,CAACD,QACnBJ,mBAAmB;gBACjB,GAAGI,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBjB;YACpB;QACFe;QACAC;IACF;AACF"}
+{"version":3,"file":"ai-model/model-adapter/chat-completion.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/model-adapter/chat-completion.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  ChatCompletionAdapter,\n  ChatCompletionCallContext,\n  ExtractContentAndReasoning,\n  MidsceneChatCompletionDefaults,\n  ModelAdapterDefinition,\n} from './types';\n\nconst defaultImageDetail = (_input: unknown) => undefined;\n\nconst defaultChatCompletionParams = ({\n  midsceneDefaults,\n  userConfig,\n}: ChatCompletionCallContext) => ({\n  config: {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n  },\n});\n\nconst midsceneChatCompletionDefaults: MidsceneChatCompletionDefaults = {\n  temperature: 0,\n};\n\nfunction createDefaultExtractContentAndReasoning(\n  reasoningContentKeys: string[],\n): ExtractContentAndReasoning {\n  return (message) => {\n    if (!message) {\n      return {\n        content: '',\n        reasoning_content: '',\n      };\n    }\n\n    const messageRecord = message as Record<string, unknown>;\n    const rawReasoning = reasoningContentKeys\n      .map((key) => messageRecord[key])\n      .find((value) => typeof value === 'string');\n    const messageReasoning =\n      typeof rawReasoning === 'string' ? rawReasoning : '';\n\n    return {\n      content: typeof message.content === 'string' ? message.content : '',\n      reasoning_content: messageReasoning,\n    };\n  };\n}\n\nfunction resolveExtractContentAndReasoning(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ExtractContentAndReasoning {\n  const messageExtraction = chatCompletion?.messageExtraction;\n  if (messageExtraction?.kind === 'custom') {\n    return messageExtraction.extractContentAndReasoning;\n  }\n\n  return createDefaultExtractContentAndReasoning(\n    messageExtraction?.reasoningContentKeys ?? ['reasoning_content'],\n  );\n}\n\nexport function resolveChatCompletion(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ChatCompletionAdapter {\n  const buildChatCompletionParams =\n    chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;\n  const resolveImageDetail =\n    chatCompletion?.resolveImageDetail ?? defaultImageDetail;\n  const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];\n  const extractContentAndReasoning =\n    resolveExtractContentAndReasoning(chatCompletion);\n  const useReasoningAsContentFallback =\n    chatCompletion?.useReasoningAsContentFallback ?? false;\n  const replayRawAssistantMessage =\n    chatCompletion?.replayRawAssistantMessage ?? false;\n\n  return {\n    unsupportedUserConfig,\n    buildChatCompletionParams: (input) => {\n      const context = {\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      };\n      return buildChatCompletionParams(context);\n    },\n    resolveImageDetail: (input) =>\n      resolveImageDetail({\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      }),\n    extractContentAndReasoning,\n    useReasoningAsContentFallback,\n    replayRawAssistantMessage,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultImageDetail","_input","undefined","defaultChatCompletionParams","midsceneDefaults","userConfig","midsceneChatCompletionDefaults","createDefaultExtractContentAndReasoning","reasoningContentKeys","message","messageRecord","rawReasoning","value","messageReasoning","resolveExtractContentAndReasoning","chatCompletion","messageExtraction","resolveChatCompletion","buildChatCompletionParams","resolveImageDetail","unsupportedUserConfig","extractContentAndReasoning","useReasoningAsContentFallback","replayRawAssistantMessage","input","context"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;ACEA,MAAMI,qBAAqB,CAACC,SAAoBC;AAEhD,MAAMC,8BAA8B,CAAC,EACnCC,gBAAgB,EAChBC,UAAU,EACgB,GAAM;QAChC,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEA,MAAME,iCAAiE;IACrE,aAAa;AACf;AAEA,SAASC,wCACPC,oBAA8B;IAE9B,OAAO,CAACC;QACN,IAAI,CAACA,SACH,OAAO;YACL,SAAS;YACT,mBAAmB;QACrB;QAGF,MAAMC,gBAAgBD;QACtB,MAAME,eAAeH,qBAClB,GAAG,CAAC,CAACb,MAAQe,aAAa,CAACf,IAAI,EAC/B,IAAI,CAAC,CAACiB,QAAU,AAAiB,YAAjB,OAAOA;QAC1B,MAAMC,mBACJ,AAAwB,YAAxB,OAAOF,eAA4BA,eAAe;QAEpD,OAAO;YACL,SAAS,AAA2B,YAA3B,OAAOF,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;YACjE,mBAAmBI;QACrB;IACF;AACF;AAEA,SAASC,kCACPC,cAAwD;IAExD,MAAMC,oBAAoBD,gBAAgB;IAC1C,IAAIC,mBAAmB,SAAS,UAC9B,OAAOA,kBAAkB,0BAA0B;IAGrD,OAAOT,wCACLS,mBAAmB,wBAAwB;QAAC;KAAoB;AAEpE;AAEO,SAASC,sBACdF,cAAwD;IAExD,MAAMG,4BACJH,gBAAgB,6BAA6BZ;IAC/C,MAAMgB,qBACJJ,gBAAgB,sBAAsBf;IACxC,MAAMoB,wBAAwBL,gBAAgB,yBAAyB,EAAE;IACzE,MAAMM,6BACJP,kCAAkCC;IACpC,MAAMO,gCACJP,gBAAgB,iCAAiC;IACnD,MAAMQ,4BACJR,gBAAgB,6BAA6B;IAE/C,OAAO;QACLK;QACA,2BAA2B,CAACI;YAC1B,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBlB;YACpB;YACA,OAAOY,0BAA0BO;QACnC;QACA,oBAAoB,CAACD,QACnBL,mBAAmB;gBACjB,GAAGK,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBlB;YACpB;QACFe;QACAC;QACAC;IACF;AACF"}

package/dist/lib/ai-model/models/kimi.js

@@ -68,6 +68,24 @@ const buildKimiChatCompletionParams = (input)=>{
         }
     };
 };
+const buildKimi3ChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    const commonOverrideConfig = {};
+    commonOverrideConfig.temperature = void 0;
+    if ('none' !== userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
+        type: 'json_object'
+    };
+    const reasoningEffort = userConfig.reasoningEffort;
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...commonOverrideConfig,
+            ...reasoningEffort ? {
+                reasoning_effort: reasoningEffort
+            } : {}
+        }
+    };
+};
 const kimiAdapters = {
     kimi: {
         chatCompletion: {
@@ -84,6 +102,23 @@ const kimiAdapters = {
                 parseRawLocateValue: parseKimiRawLocateValue
             }
         }
+    },
+    kimi3: {
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningEnabled',
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: buildKimi3ChatCompletionParams,
+            useReasoningAsContentFallback: true,
+            replayRawAssistantMessage: true
+        },
+        locate: {
+            resultAdapter: {
+                coordinates: kimiNormalizedPointCoordinatesMeta,
+                parseRawLocateValue: parseKimiRawLocateValue
+            }
+        }
     }
 };
 exports.kimiAdapters = __webpack_exports__.kimiAdapters;

package/dist/lib/ai-model/models/kimi.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/kimi.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  parseCoordinateList,\n} from '../shared/model-locate-result';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1,\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n  const point = parseCoordinateList(input, 'point');\n  if (point.length < 2) {\n    throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n  }\n  const [x, y] = point;\n  // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n  // normalized coordinates, otherwise they are treated as screenshot pixels.\n  const coordinatesMeta =\n    x <= 1 && y <= 1\n      ? kimiNormalizedPointCoordinatesMeta\n      : kimiPixelPointCoordinatesMeta;\n  return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const effectiveReasoningEnabled = reasoningEnabled ?? false;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // kimi disallow custom temperature\n  commonOverrideConfig.temperature = undefined;\n\n  // Kimi Chat Completions response_format:\n  // https://platform.kimi.com/docs/api/chat\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const kimiAdapters = {\n  kimi: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimiChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: kimiNormalizedPointCoordinatesMeta,\n        parseRawLocateValue: parseKimiRawLocateValue,\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'kimi'>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","kimiAdapters"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACMA,MAAMI,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;AACT;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,AAAAA,IAAAA,yBAAAA,mBAAAA,AAAAA,EAAoBF,OAAO;IACzC,IAAIC,MAAM,MAAM,GAAG,GACjB,MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,AAAAA,IAAAA,yBAAAA,uBAAAA,AAAAA,EAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAInC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C;QACnD,UAAU;YACR,MAAMH,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BR;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAaZ;gBACb,qBAAqBE;YACvB;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/models/kimi.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  parseCoordinateList,\n} from '../shared/model-locate-result';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1,\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n  const point = parseCoordinateList(input, 'point');\n  if (point.length < 2) {\n    throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n  }\n  const [x, y] = point;\n  // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n  // normalized coordinates, otherwise they are treated as screenshot pixels.\n  const coordinatesMeta =\n    x <= 1 && y <= 1\n      ? kimiNormalizedPointCoordinatesMeta\n      : kimiPixelPointCoordinatesMeta;\n  return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const effectiveReasoningEnabled = reasoningEnabled ?? false;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // kimi disallow custom temperature\n  commonOverrideConfig.temperature = undefined;\n\n  // Kimi Chat Completions response_format:\n  // https://platform.kimi.com/docs/api/chat\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst buildKimi3ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // Kimi disallows custom temperature.\n  commonOverrideConfig.temperature = undefined;\n\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  // Kimi K3 currently only supports reasoning_effort=\"max\"; its docs say\n  // additional effort levels will be available later.\n  const reasoningEffort = userConfig.reasoningEffort;\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}),\n    },\n  };\n};\n\nexport const kimiAdapters = {\n  kimi: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimiChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: kimiNormalizedPointCoordinatesMeta,\n        parseRawLocateValue: parseKimiRawLocateValue,\n      },\n    },\n  },\n  kimi3: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEnabled', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimi3ChatCompletionParams,\n      useReasoningAsContentFallback: true,\n      // Kimi K3 multi-turn and tool calls must replay the complete assistant\n      // message, including reasoning_content and tool_calls.\n      // https://platform.kimi.com/docs/guide/use-thinking-effort\n      replayRawAssistantMessage: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: kimiNormalizedPointCoordinatesMeta,\n        parseRawLocateValue: parseKimiRawLocateValue,\n      },\n    },\n  },\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'kimi' | 'kimi3'\n>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","buildKimi3ChatCompletionParams","reasoningEffort","kimiAdapters"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACMA,MAAMI,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;AACT;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,AAAAA,IAAAA,yBAAAA,mBAAAA,AAAAA,EAAoBF,OAAO;IACzC,IAAIC,MAAM,MAAM,GAAG,GACjB,MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,AAAAA,IAAAA,yBAAAA,uBAAAA,AAAAA,EAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAInC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C;QACnD,UAAU;YACR,MAAMH,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,iCAAiC,CACrCjB;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAMc,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAEnC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAK/D,MAAMI,kBAAkBP,WAAW,eAAe;IAElD,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAII,kBAAkB;gBAAE,kBAAkBA;YAAgB,IAAI,CAAC,CAAC;QAClE;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BV;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAaZ;gBACb,qBAAqBE;YACvB;QACF;IACF;IACA,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBAAC;gBAAoB;aAAkB;YAC9D,2BAA2BkB;YAC3B,+BAA+B;YAI/B,2BAA2B;QAC7B;QACA,QAAQ;YACN,eAAe;gBACb,aAAapB;gBACb,qBAAqBE;YACvB;QACF;IACF;AACF"}

package/dist/lib/ai-model/prompt/extraction.js

@@ -166,14 +166,16 @@ By viewing the screenshot and page contents, you can extract the following data:
 </data-json>
 `;
 }
-const extractDataQueryPrompt = (pageDescription, dataQuery)=>{
+const extractDataQueryPrompt = (pageDescription, dataQuery, context)=>{
     let dataQueryText = '';
     dataQueryText = 'string' == typeof dataQuery ? dataQuery : JSON.stringify(dataQuery, null, 2);
+    const trimmedContext = context?.trim();
+    const contextSection = trimmedContext ? `\n<CONTEXT>\n${trimmedContext}\n</CONTEXT>\n` : '';
     return `
 <PageDescription>
 ${pageDescription}
 </PageDescription>
-
+${contextSection}
 <DATA_DEMAND>
 ${dataQueryText}
 </DATA_DEMAND>