@aiscene/core 9.0.0 → 9.0.2

package/dist/lib/agent/ui-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/ui-utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Act'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      // Prefer AI-generated output.log over user input\n      const planTask = task as ExecutionTaskPlanning;\n      value = planTask.output?.log || planTask.param?.userInstruction;\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'PullGesture'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    // Flatten `{key: \"raw value\"}` into `key: raw value` instead of emitting\n    // a pretty-printed JSON string. JSON.stringify would escape every inner\n    // quote as `\\\"`, and the UI renders the result as plain text — so a\n    // command like `grep -E \"version\"` ends up shown with the literal\n    // backslashes, which is both noisy and confusing for users.\n    const entries = Object.entries(value as Record<string, unknown>);\n    if (entries.length === 0) {\n      return '';\n    }\n    const formatValue = (v: unknown): string => {\n      if (typeof v === 'string') return v;\n      if (v === null || v === undefined) return String(v);\n      if (typeof v === 'object') return JSON.stringify(v);\n      return String(v);\n    };\n    if (entries.length === 1) {\n      const [key, v] = entries[0];\n      return `${key}: ${formatValue(v)}`;\n    }\n    return entries.map(([key, v]) => `${key}: ${formatValue(v)}`).join(', ');\n  }\n\n  return String(value);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","planTask","locateStr","entries","formatValue","v","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,MAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACMO,SAASI,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAE9B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC,SACdA,QAAQ,kBAAkB,UAC/BC,MAAM,OAAO,CAACD,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BE;IAIN,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIN,UAAU,MAAM,EAClB,OAAO;QACL,SAASK,UAAUL,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASK,UAAUL,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAEQ,UAAU,EAAE,GAAGR;QAEvB,IAAI,AAAsB,YAAtB,OAAOQ,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQP,cAAcO;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAEO,SAASC,aACdC,IAcY,EACZjB,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGiB,KAAK,GAAG,EAAEjB,QAAQ;IAE9B,OAAOiB;AACT;AAEO,SAASC,SAASrB,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,MAAsC;SACzD;QAEL,MAAMsB,WAAWtB;QACjBgB,QAAQM,SAAS,MAAM,EAAE,OAAOA,SAAS,KAAK,EAAE;IAClD;IAGF,IAAItB,AAAc,cAAdA,KAAK,IAAI,EACXgB,QAAQP,oBAAqBT,MAAc,OAAO,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;QAChC,MAAME,SAAUF,MAA8B,OAAO;QACrD,MAAMuB,YAAYrB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,MAA8B,OAAO,QAC/CgB,QAAQ,GAAIhB,MAA8B,OAAO,OAAO,EAAE,CAAC;aACtD,IACL,AAA4D,YAA5D,OAAQA,MAA8B,OAAO,YAE7CgB,QAAQZ,eAAgBJ,MAA8B;aACjD,IACL,AAA2D,YAA3D,OAAQA,MAA8B,OAAO,aAC5CA,MAA8B,YAAY,eAE3CgB,QAAQV,aAAcN,MAA8B;aAC/C,IACL,AAAuD,WAA/CA,MAA8B,OAAO,OAE7CgB,QAAShB,MAA8B,OAAO;aACzC,IACJA,MAA8B,SAC/B,AAAgD,YAAhD,OAAQA,MAA8B,SACtCL,OAAO,IAAI,CAAEK,MAA8B,SAAS,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,MAA8B;QAGzC,IAAIuB,WAEAP,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGO,UAAU,GAAG,EAAEP,OAAO,GAEzBO;IAGd;IAEA,IAAI,AAAiB,WAAVP,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMO,YAAYtB,eAAee;QACjC,IAAIO,WACF,OAAOA;QAOT,MAAMC,UAAU7B,OAAO,OAAO,CAACqB;QAC/B,IAAIQ,AAAmB,MAAnBA,QAAQ,MAAM,EAChB,OAAO;QAET,MAAMC,cAAc,CAACC;YACnB,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOA;YAClC,IAAIA,QAAAA,GAA+B,OAAOC,OAAOD;YACjD,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOT,KAAK,SAAS,CAACS;YACjD,OAAOC,OAAOD;QAChB;QACA,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;YACxB,MAAM,CAAC9B,KAAKgC,EAAE,GAAGF,OAAO,CAAC,EAAE;YAC3B,OAAO,GAAG9B,IAAI,EAAE,EAAE+B,YAAYC,IAAI;QACpC;QACA,OAAOF,QAAQ,GAAG,CAAC,CAAC,CAAC9B,KAAKgC,EAAE,GAAK,GAAGhC,IAAI,EAAE,EAAE+B,YAAYC,IAAI,EAAE,IAAI,CAAC;IACrE;IAEA,OAAOC,OAAOX;AAChB"}
+{"version":3,"file":"agent/ui-utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/ui-utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Act'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      // Prefer AI-generated output.log over user input\n      const planTask = task as ExecutionTaskPlanning;\n      value = planTask.output?.log || planTask.param?.userInstruction;\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'PullGesture'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    // Flatten `{key: \"raw value\"}` into `key: raw value` instead of emitting\n    // a pretty-printed JSON string. JSON.stringify would escape every inner\n    // quote as `\\\"`, and the UI renders the result as plain text — so a\n    // command like `grep -E \"version\"` ends up shown with the literal\n    // backslashes, which is both noisy and confusing for users.\n    const entries = Object.entries(value as Record<string, unknown>);\n    if (entries.length === 0) {\n      return '';\n    }\n    const formatValue = (v: unknown): string => {\n      if (typeof v === 'string') return v;\n      if (v === null || v === undefined) return String(v);\n      if (typeof v === 'object') return JSON.stringify(v);\n      return String(v);\n    };\n    if (entries.length === 1) {\n      const [key, v] = entries[0];\n      return `${key}: ${formatValue(v)}`;\n    }\n    return entries.map(([key, v]) => `${key}: ${formatValue(v)}`).join(', ');\n  }\n\n  return String(value);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","planTask","locateStr","entries","formatValue","v","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACMO,SAASI,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAE9B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC,SACdA,QAAQ,kBAAkB,UAC/BC,MAAM,OAAO,CAACD,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BE;IAIN,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIN,UAAU,MAAM,EAClB,OAAO;QACL,SAASK,UAAUL,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASK,UAAUL,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAEQ,UAAU,EAAE,GAAGR;QAEvB,IAAI,AAAsB,YAAtB,OAAOQ,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQP,cAAcO;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAEO,SAASC,aACdC,IAcY,EACZjB,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGiB,KAAK,GAAG,EAAEjB,QAAQ;IAE9B,OAAOiB;AACT;AAEO,SAASC,SAASrB,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,MAAsC;SACzD;QAEL,MAAMsB,WAAWtB;QACjBgB,QAAQM,SAAS,MAAM,EAAE,OAAOA,SAAS,KAAK,EAAE;IAClD;IAGF,IAAItB,AAAc,cAAdA,KAAK,IAAI,EACXgB,QAAQP,oBAAqBT,MAAc,OAAO,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;QAChC,MAAME,SAAUF,MAA8B,OAAO;QACrD,MAAMuB,YAAYrB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,MAA8B,OAAO,QAC/CgB,QAAQ,GAAIhB,MAA8B,OAAO,OAAO,EAAE,CAAC;aACtD,IACL,AAA4D,YAA5D,OAAQA,MAA8B,OAAO,YAE7CgB,QAAQZ,eAAgBJ,MAA8B;aACjD,IACL,AAA2D,YAA3D,OAAQA,MAA8B,OAAO,aAC5CA,MAA8B,YAAY,eAE3CgB,QAAQV,aAAcN,MAA8B;aAC/C,IACL,AAAuD,WAA/CA,MAA8B,OAAO,OAE7CgB,QAAShB,MAA8B,OAAO;aACzC,IACJA,MAA8B,SAC/B,AAAgD,YAAhD,OAAQA,MAA8B,SACtCL,OAAO,IAAI,CAAEK,MAA8B,SAAS,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,MAA8B;QAGzC,IAAIuB,WAEAP,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGO,UAAU,GAAG,EAAEP,OAAO,GAEzBO;IAGd;IAEA,IAAI,AAAiB,WAAVP,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMO,YAAYtB,eAAee;QACjC,IAAIO,WACF,OAAOA;QAOT,MAAMC,UAAU7B,OAAO,OAAO,CAACqB;QAC/B,IAAIQ,AAAmB,MAAnBA,QAAQ,MAAM,EAChB,OAAO;QAET,MAAMC,cAAc,CAACC;YACnB,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOA;YAClC,IAAIA,QAAAA,GAA+B,OAAOC,OAAOD;YACjD,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOT,KAAK,SAAS,CAACS;YACjD,OAAOC,OAAOD;QAChB;QACA,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;YACxB,MAAM,CAAC9B,KAAKgC,EAAE,GAAGF,OAAO,CAAC,EAAE;YAC3B,OAAO,GAAG9B,IAAI,EAAE,EAAE+B,YAAYC,IAAI;QACpC;QACA,OAAOF,QAAQ,GAAG,CAAC,CAAC,CAAC9B,KAAKgC,EAAE,GAAK,GAAGhC,IAAI,EAAE,EAAE+B,YAAYC,IAAI,EAAE,IAAI,CAAC;IACrE;IAEA,OAAOC,OAAOX;AAChB"}

package/dist/lib/agent/usage-intent.js

@@ -13,7 +13,7 @@ var __webpack_require__ = {};
 })();
 (()=>{
     __webpack_require__.r = (exports1)=>{
-        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
             value: 'Module'
         });
         Object.defineProperty(exports1, '__esModule', {

package/dist/lib/agent/usage-intent.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/usage-intent.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/usage-intent.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { AIUsageInfo } from '@/types';\nimport type { TIntent } from '@aiscene/shared/env';\nimport { getDebug } from '@aiscene/shared/logger';\n\nconst warnUsageIntent = getDebug('agent:usage-intent', { console: true });\n\nexport function withUsageIntent(\n  usage: AIUsageInfo | undefined,\n  intent: TIntent,\n): AIUsageInfo | undefined {\n  if (!usage) {\n    return undefined;\n  }\n\n  if (usage.intent) {\n    warnUsageIntent(\n      `intent is already set to \"${usage.intent}\", skipping overwrite to \"${intent}\"`,\n    );\n    return usage;\n  }\n\n  return {\n    ...usage,\n    intent,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","warnUsageIntent","getDebug","withUsageIntent","usage","intent"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,MAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACFA,MAAMI,kBAAkBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,sBAAsB;IAAE,SAAS;AAAK;AAEhE,SAASC,gBACdC,KAA8B,EAC9BC,MAAe;IAEf,IAAI,CAACD,OACH;IAGF,IAAIA,MAAM,MAAM,EAAE;QAChBH,gBACE,CAAC,0BAA0B,EAAEG,MAAM,MAAM,CAAC,0BAA0B,EAAEC,OAAO,CAAC,CAAC;QAEjF,OAAOD;IACT;IAEA,OAAO;QACL,GAAGA,KAAK;QACRC;IACF;AACF"}
+{"version":3,"file":"agent/usage-intent.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/usage-intent.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { AIUsageInfo } from '@/types';\nimport type { TIntent } from '@aiscene/shared/env';\nimport { getDebug } from '@aiscene/shared/logger';\n\nconst warnUsageIntent = getDebug('agent:usage-intent', { console: true });\n\nexport function withUsageIntent(\n  usage: AIUsageInfo | undefined,\n  intent: TIntent,\n): AIUsageInfo | undefined {\n  if (!usage) {\n    return undefined;\n  }\n\n  if (usage.intent) {\n    warnUsageIntent(\n      `intent is already set to \"${usage.intent}\", skipping overwrite to \"${intent}\"`,\n    );\n    return usage;\n  }\n\n  return {\n    ...usage,\n    intent,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","warnUsageIntent","getDebug","withUsageIntent","usage","intent"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACFA,MAAMI,kBAAkBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,sBAAsB;IAAE,SAAS;AAAK;AAEhE,SAASC,gBACdC,KAA8B,EAC9BC,MAAe;IAEf,IAAI,CAACD,OACH;IAGF,IAAIA,MAAM,MAAM,EAAE;QAChBH,gBACE,CAAC,0BAA0B,EAAEG,MAAM,MAAM,CAAC,0BAA0B,EAAEC,OAAO,CAAC,CAAC;QAEjF,OAAOD;IACT;IAEA,OAAO;QACL,GAAGA,KAAK;QACRC;IACF;AACF"}

package/dist/lib/agent/utils.js

@@ -22,7 +22,7 @@ var __webpack_require__ = {};
 })();
 (()=>{
     __webpack_require__.r = (exports1)=>{
-        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
             value: 'Module'
         });
         Object.defineProperty(exports1, '__esModule', {
@@ -170,7 +170,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"9.0.0";
+const getMidsceneVersion = ()=>"9.0.2";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/agent/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport type { TModelFamily } from '@aiscene/shared/env';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@aiscene/shared/env';\nimport { generateElementByRect } from '@aiscene/shared/extractor';\nimport { imageInfoOfBase64, resizeImgBase64 } from '@aiscene/shared/img';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { _keyDefinitions } from '@aiscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@aiscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n    modelFamily?: TModelFamily;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    // Convert bbox [x1, y1, x2, y2] to rect {left, top, width, height}\n    const rect = {\n      left: planLocateParam.bbox[0],\n      top: planLocateParam.bbox[1],\n      width: planLocateParam.bbox[2] - planLocateParam.bbox[0] + 1,\n      height: planLocateParam.bbox[3] - planLocateParam.bbox[1] + 1,\n    };\n\n    const element = generateElementByRect(\n      rect,\n      typeof planLocateParam.prompt === 'string'\n        ? planLocateParam.prompt\n        : planLocateParam.prompt?.prompt || '',\n    );\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","commonContextParser","interfaceInstance","_opt","debug","getDebug","assert","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotBase64","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","rect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,MAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;ACoBO,eAAeI,oBACpBC,iBAAoC,EACpCC,IAIC;IAED,MAAMC,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;IAEvBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOJ,mBAAmB;IAE1BE,MAAM;IACN,MAAMG,cAAcL,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNI,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuB;QACrB,SAASD;QACT,WAAWJ,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMK,gBAAgB,MAAMP,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOQ,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIP,MAAM,CAAC,MAAM,EAAEM,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMI,mBAAmB,MAAMb,kBAAkB,gBAAgB;IACjE,MAAMc,uBAAuBC,KAAK,GAAG;IACrCX,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOS,kBAAmB;IAG1BX,MAAM;IACN,MAAM,EAAE,OAAOc,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,AAAAA,IAAAA,oBAAAA,iBAAAA,AAAAA,EAAkBL;IAE1B,IAAI,CAACF,OAAO,QAAQ,CAACK,aAAa,CAACL,OAAO,QAAQ,CAACM,YACjD,MAAM,IAAIL,MACR,CAAC,wFAAwF,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIL,MACR,CAAC,0FAA0F,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAGjIf,MAAM,yBAAyBc,UAAU,KAAKC;IAM9C,MAAME,oBAAoBX,eAAeC;IACzC,MAAMW,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBb;IAExB,IAAIW,sBAAsBC,sBAAsB;QAC9ClB,MACE,CAAC,4CAA4C,EAAEM,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEU,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBZ;IAEtB;IAEA,MAAMa,mBAAmBrB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACU,OAAO,QAAQ,CAACW,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIV,MACR,CAAC,wEAAwE,EAAEU,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBnB,MAAM,mBAAmBqB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCpB,MAAM,4BAA4BsB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CpB,MACE,CAAC,mCAAmC,EAAEoB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBhB,kBAAkB;YAC5D,OAAOY;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACF,eAAed;YACjDU;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOR;YACP,QAAQC;QACV;QACA,eAAeM;QACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACjB,kBAAkBC;QACpDU;IACF;AACF;AAEO,SAASO,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAAA,mBAAAA,CAAAA,iBAAqC,CACzDC,oBAAAA,wBAAwBA;IAE1B,MAAMC,qBAAqBC,2BAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,IAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAAA,mBAAAA,CAAAA,qBAAyC,CAACQ,oBAAAA,qBAAqBA,GACjE;IAEFC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAEO,SAASG,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;QAExB,MAAMG,OAAO;YACX,MAAMH,gBAAgB,IAAI,CAAC,EAAE;YAC7B,KAAKA,gBAAgB,IAAI,CAAC,EAAE;YAC5B,OAAOA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;YAC3D,QAAQA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;QAC9D;QAEA,MAAMI,UAAUC,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EACdF,MACA,AAAkC,YAAlC,OAAOH,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QAExC,OAAOI;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EACE;IAKJ,IAAI;QACF,MAAMR,OACJ,MAAMI,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNvB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCtB,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOM,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAzB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOyB;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNvB,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;YAC/BE,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;SAChC;QACD,MAAM;YACJ,GAAGyB,QAAQ,IAAI;YACf,MAAMvB,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,IAAI,GAAGzB;YACrC,KAAKE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,GAAG,GAAGzB;YACnC,OAAOE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,KAAK,GAAGzB;YACvC,QAAQE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,MAAM,GAAGzB;QAC3C;IACF;AACF;AAEO,MAAMwC,uCAAuC,CAClDhB,MACAxB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOwB;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMtB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGxB;QAC7B,KAAKE,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGxB;QAC3B,OAAOE,KAAK,KAAK,CAACsB,KAAK,KAAK,GAAGxB;QAC/B,QAAQE,KAAK,KAAK,CAACsB,KAAK,MAAM,GAAGxB;IACnC;AACF"}
+{"version":3,"file":"agent/utils.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport type { TModelFamily } from '@aiscene/shared/env';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@aiscene/shared/env';\nimport { generateElementByRect } from '@aiscene/shared/extractor';\nimport { imageInfoOfBase64, resizeImgBase64 } from '@aiscene/shared/img';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { _keyDefinitions } from '@aiscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@aiscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n    modelFamily?: TModelFamily;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    // Convert bbox [x1, y1, x2, y2] to rect {left, top, width, height}\n    const rect = {\n      left: planLocateParam.bbox[0],\n      top: planLocateParam.bbox[1],\n      width: planLocateParam.bbox[2] - planLocateParam.bbox[0] + 1,\n      height: planLocateParam.bbox[3] - planLocateParam.bbox[1] + 1,\n    };\n\n    const element = generateElementByRect(\n      rect,\n      typeof planLocateParam.prompt === 'string'\n        ? planLocateParam.prompt\n        : planLocateParam.prompt?.prompt || '',\n    );\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","commonContextParser","interfaceInstance","_opt","debug","getDebug","assert","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotBase64","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","rect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;ACoBO,eAAeI,oBACpBC,iBAAoC,EACpCC,IAIC;IAED,MAAMC,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;IAEvBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOJ,mBAAmB;IAE1BE,MAAM;IACN,MAAMG,cAAcL,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNI,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuB;QACrB,SAASD;QACT,WAAWJ,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMK,gBAAgB,MAAMP,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOQ,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIP,MAAM,CAAC,MAAM,EAAEM,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMI,mBAAmB,MAAMb,kBAAkB,gBAAgB;IACjE,MAAMc,uBAAuBC,KAAK,GAAG;IACrCX,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOS,kBAAmB;IAG1BX,MAAM;IACN,MAAM,EAAE,OAAOc,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,AAAAA,IAAAA,oBAAAA,iBAAAA,AAAAA,EAAkBL;IAE1B,IAAI,CAACF,OAAO,QAAQ,CAACK,aAAa,CAACL,OAAO,QAAQ,CAACM,YACjD,MAAM,IAAIL,MACR,CAAC,wFAAwF,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIL,MACR,CAAC,0FAA0F,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAGjIf,MAAM,yBAAyBc,UAAU,KAAKC;IAM9C,MAAME,oBAAoBX,eAAeC;IACzC,MAAMW,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBb;IAExB,IAAIW,sBAAsBC,sBAAsB;QAC9ClB,MACE,CAAC,4CAA4C,EAAEM,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEU,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBZ;IAEtB;IAEA,MAAMa,mBAAmBrB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACU,OAAO,QAAQ,CAACW,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIV,MACR,CAAC,wEAAwE,EAAEU,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBnB,MAAM,mBAAmBqB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCpB,MAAM,4BAA4BsB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CpB,MACE,CAAC,mCAAmC,EAAEoB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBhB,kBAAkB;YAC5D,OAAOY;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACF,eAAed;YACjDU;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOR;YACP,QAAQC;QACV;QACA,eAAeM;QACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACjB,kBAAkBC;QACpDU;IACF;AACF;AAEO,SAASO,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAAA,mBAAAA,CAAAA,iBAAqC,CACzDC,oBAAAA,wBAAwBA;IAE1B,MAAMC,qBAAqBC,2BAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,IAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAAA,mBAAAA,CAAAA,qBAAyC,CAACQ,oBAAAA,qBAAqBA,GACjE;IAEFC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAEO,SAASG,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;QAExB,MAAMG,OAAO;YACX,MAAMH,gBAAgB,IAAI,CAAC,EAAE;YAC7B,KAAKA,gBAAgB,IAAI,CAAC,EAAE;YAC5B,OAAOA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;YAC3D,QAAQA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;QAC9D;QAEA,MAAMI,UAAUC,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EACdF,MACA,AAAkC,YAAlC,OAAOH,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QAExC,OAAOI;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EACE;IAKJ,IAAI;QACF,MAAMR,OACJ,MAAMI,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNvB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCtB,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOM,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAzB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOyB;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNvB,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;YAC/BE,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;SAChC;QACD,MAAM;YACJ,GAAGyB,QAAQ,IAAI;YACf,MAAMvB,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,IAAI,GAAGzB;YACrC,KAAKE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,GAAG,GAAGzB;YACnC,OAAOE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,KAAK,GAAGzB;YACvC,QAAQE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,MAAM,GAAGzB;QAC3C;IACF;AACF;AAEO,MAAMwC,uCAAuC,CAClDhB,MACAxB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOwB;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMtB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGxB;QAC7B,KAAKE,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGxB;QAC3B,OAAOE,KAAK,KAAK,CAACsB,KAAK,KAAK,GAAGxB;QAC/B,QAAQE,KAAK,KAAK,CAACsB,KAAK,MAAM,GAAGxB;IACnC;AACF"}

package/dist/lib/ai-model/auto-glm/actions.js

@@ -13,7 +13,7 @@ var __webpack_require__ = {};
 })();
 (()=>{
     __webpack_require__.r = (exports1)=>{
-        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
             value: 'Module'
         });
         Object.defineProperty(exports1, '__esModule', {

package/dist/lib/ai-model/auto-glm/actions.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/auto-glm/actions.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/actions.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { adaptBbox, pointToBbox } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\n\nconst debug = getDebug('auto-glm-actions');\n\n/**\n * Auto-GLM coordinate system range: [0, AUTO_GLM_COORDINATE_MAX]\n */\nconst AUTO_GLM_COORDINATE_MAX = 1000;\n\n/**\n * Convert auto-glm coordinate [0,1000] to bbox in pixel coordinates\n */\nfunction autoGLMCoordinateToBbox(\n  x: number,\n  y: number,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const bbox = pointToBbox(x, y, 10);\n  return adaptBbox(bbox, width, height, 'auto-glm');\n}\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type ParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nconst BACK_BUTTON_NAMES = ['AndroidBackButton', 'HarmonyBackButton'];\nconst HOME_BUTTON_NAMES = ['AndroidHomeButton', 'HarmonyHomeButton'];\n\n/**\n * Find the action name in actionSpace that matches one of the known names.\n * Falls back to defaultName if no match found or actionSpace is not provided.\n */\nfunction findActionName(\n  actionSpace: DeviceAction[] | undefined,\n  knownNames: string[],\n  defaultName: string,\n): string {\n  if (!actionSpace) return defaultName;\n  const match = actionSpace.find((a) => knownNames.includes(a.name));\n  return match ? match.name : defaultName;\n}\n\nexport function transformAutoGLMAction(\n  action: ParsedAction,\n  size: { width: number; height: number },\n  actionSpace?: DeviceAction[],\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              tapAction.element[0],\n              tapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              doubleTapAction.element[0],\n              doubleTapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            // Calculate locate using start coordinate\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              swipeAction.start[0],\n              swipeAction.start[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            // Calculate horizontal and vertical delta in [0,AUTO_GLM_COORDINATE_MAX] coordinate system\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = Math.round(\n                (absDeltaY * size.height) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = Math.round(\n                (absDeltaX * size.width) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate,\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              },\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              longPressAction.element[0],\n              longPressAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'LongPress',\n                param: {\n                  locate,\n                },\n                thought: longPressAction.think || '',\n              },\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  BACK_BUTTON_NAMES,\n                  'AndroidBackButton',\n                ),\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  HOME_BUTTON_NAMES,\n                  'AndroidHomeButton',\n                ),\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: { uri: launchAction.app },\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","AUTO_GLM_COORDINATE_MAX","autoGLMCoordinateToBbox","x","y","width","height","bbox","pointToBbox","adaptBbox","BACK_BUTTON_NAMES","HOME_BUTTON_NAMES","findActionName","actionSpace","knownNames","defaultName","match","a","transformAutoGLMAction","action","size","finishAction","doAction","tapAction","x1","y1","x2","y2","locate","doubleTapAction","typeAction","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,MAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAKvB,MAAMC,0BAA0B;AAKhC,SAASC,wBACPC,CAAS,EACTC,CAAS,EACTC,KAAa,EACbC,MAAc;IAEd,MAAMC,OAAOC,AAAAA,IAAAA,mCAAAA,WAAAA,AAAAA,EAAYL,GAAGC,GAAG;IAC/B,OAAOK,AAAAA,IAAAA,mCAAAA,SAAAA,AAAAA,EAAUF,MAAMF,OAAOC,QAAQ;AACxC;AAwGA,MAAMI,oBAAoB;IAAC;IAAqB;CAAoB;AACpE,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AAMpE,SAASC,eACPC,WAAuC,EACvCC,UAAoB,EACpBC,WAAmB;IAEnB,IAAI,CAACF,aAAa,OAAOE;IACzB,MAAMC,QAAQH,YAAY,IAAI,CAAC,CAACI,IAAMH,WAAW,QAAQ,CAACG,EAAE,IAAI;IAChE,OAAOD,QAAQA,MAAM,IAAI,GAAGD;AAC9B;AAEO,SAASG,uBACdC,MAAoB,EACpBC,IAAuC,EACvCP,WAA4B;IAE5B,IAAI;QACF,OAAQM,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAME,eAAeF;oBACrBpB,MAAM,4BAA4BsB;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWH;oBAejB,OAASG,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBvB,MAAM,yBAAyBwB;gCAC/B,MAAM,CAACC,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvBqB,UAAU,OAAO,CAAC,EAAE,EACpBA,UAAU,OAAO,CAAC,EAAE,EACpBH,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBP;gCACxBvB,MAAM,gCAAgC8B;gCACtC,MAAM,CAACL,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvB2B,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BT,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAME,aAAaR;gCACnBvB,MAAM,0BAA0B+B;gCAEhC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOA,WAAW,IAAI;wCACxB;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAMC,cAAcT;gCACpBvB,MAAM,2BAA2BgC;gCAGjC,MAAM,CAACP,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvB6B,YAAY,KAAK,CAAC,EAAE,EACpBA,YAAY,KAAK,CAAC,EAAE,EACpBX,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAGA,MAAMK,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWE,KAAK,KAAK,CAClBC,YAAYlB,KAAK,MAAM,GAAInB;oCAE9BiC,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWE,KAAK,KAAK,CAClBD,YAAYhB,KAAK,KAAK,GAAInB;oCAE7BiC,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEAjC,MACE,CAAC,2BAA2B,EAAEmC,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLP;4CAEAO;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMQ,kBAAkBjB;gCACxBvB,MAAM,gCAAgCwC;gCACtC,MAAM,CAACf,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvBqC,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BnB,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;wCACA,SAASW,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAalB;gCACnBvB,MAAM,0BAA0ByC;gCAChC,OAAO;oCACL;wCACE,MAAM5B,eACJC,aACAH,mBACA;wCAEF,OAAO,CAAC;wCACR,SAAS8B,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAanB;gCACnBvB,MAAM,0BAA0B0C;gCAChC,OAAO;oCACL;wCACE,MAAM7B,eACJC,aACAF,mBACA;wCAEF,OAAO,CAAC;wCACR,SAAS8B,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAapB;gCACnBvB,MAAM,0BAA0B2C;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAerB;gCACrBvB,MAAM,4BAA4B4C;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CAAE,KAAKA,aAAa,GAAG;wCAAC;wCAC/B,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGtB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIsB,MACR,CAAC,yBAAyB,EAAGzB,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAO0B,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrE9C,MAAM,oBAAoB+C;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}
+{"version":3,"file":"ai-model/auto-glm/actions.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/actions.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { adaptBbox, pointToBbox } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\n\nconst debug = getDebug('auto-glm-actions');\n\n/**\n * Auto-GLM coordinate system range: [0, AUTO_GLM_COORDINATE_MAX]\n */\nconst AUTO_GLM_COORDINATE_MAX = 1000;\n\n/**\n * Convert auto-glm coordinate [0,1000] to bbox in pixel coordinates\n */\nfunction autoGLMCoordinateToBbox(\n  x: number,\n  y: number,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const bbox = pointToBbox(x, y, 10);\n  return adaptBbox(bbox, width, height, 'auto-glm');\n}\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type ParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nconst BACK_BUTTON_NAMES = ['AndroidBackButton', 'HarmonyBackButton'];\nconst HOME_BUTTON_NAMES = ['AndroidHomeButton', 'HarmonyHomeButton'];\n\n/**\n * Find the action name in actionSpace that matches one of the known names.\n * Falls back to defaultName if no match found or actionSpace is not provided.\n */\nfunction findActionName(\n  actionSpace: DeviceAction[] | undefined,\n  knownNames: string[],\n  defaultName: string,\n): string {\n  if (!actionSpace) return defaultName;\n  const match = actionSpace.find((a) => knownNames.includes(a.name));\n  return match ? match.name : defaultName;\n}\n\nexport function transformAutoGLMAction(\n  action: ParsedAction,\n  size: { width: number; height: number },\n  actionSpace?: DeviceAction[],\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              tapAction.element[0],\n              tapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              doubleTapAction.element[0],\n              doubleTapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            // Calculate locate using start coordinate\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              swipeAction.start[0],\n              swipeAction.start[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            // Calculate horizontal and vertical delta in [0,AUTO_GLM_COORDINATE_MAX] coordinate system\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = Math.round(\n                (absDeltaY * size.height) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = Math.round(\n                (absDeltaX * size.width) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate,\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              },\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              longPressAction.element[0],\n              longPressAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'LongPress',\n                param: {\n                  locate,\n                },\n                thought: longPressAction.think || '',\n              },\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  BACK_BUTTON_NAMES,\n                  'AndroidBackButton',\n                ),\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  HOME_BUTTON_NAMES,\n                  'AndroidHomeButton',\n                ),\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: { uri: launchAction.app },\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","AUTO_GLM_COORDINATE_MAX","autoGLMCoordinateToBbox","x","y","width","height","bbox","pointToBbox","adaptBbox","BACK_BUTTON_NAMES","HOME_BUTTON_NAMES","findActionName","actionSpace","knownNames","defaultName","match","a","transformAutoGLMAction","action","size","finishAction","doAction","tapAction","x1","y1","x2","y2","locate","doubleTapAction","typeAction","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAKvB,MAAMC,0BAA0B;AAKhC,SAASC,wBACPC,CAAS,EACTC,CAAS,EACTC,KAAa,EACbC,MAAc;IAEd,MAAMC,OAAOC,AAAAA,IAAAA,mCAAAA,WAAAA,AAAAA,EAAYL,GAAGC,GAAG;IAC/B,OAAOK,AAAAA,IAAAA,mCAAAA,SAAAA,AAAAA,EAAUF,MAAMF,OAAOC,QAAQ;AACxC;AAwGA,MAAMI,oBAAoB;IAAC;IAAqB;CAAoB;AACpE,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AAMpE,SAASC,eACPC,WAAuC,EACvCC,UAAoB,EACpBC,WAAmB;IAEnB,IAAI,CAACF,aAAa,OAAOE;IACzB,MAAMC,QAAQH,YAAY,IAAI,CAAC,CAACI,IAAMH,WAAW,QAAQ,CAACG,EAAE,IAAI;IAChE,OAAOD,QAAQA,MAAM,IAAI,GAAGD;AAC9B;AAEO,SAASG,uBACdC,MAAoB,EACpBC,IAAuC,EACvCP,WAA4B;IAE5B,IAAI;QACF,OAAQM,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAME,eAAeF;oBACrBpB,MAAM,4BAA4BsB;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWH;oBAejB,OAASG,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBvB,MAAM,yBAAyBwB;gCAC/B,MAAM,CAACC,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvBqB,UAAU,OAAO,CAAC,EAAE,EACpBA,UAAU,OAAO,CAAC,EAAE,EACpBH,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBP;gCACxBvB,MAAM,gCAAgC8B;gCACtC,MAAM,CAACL,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvB2B,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BT,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAME,aAAaR;gCACnBvB,MAAM,0BAA0B+B;gCAEhC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOA,WAAW,IAAI;wCACxB;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAMC,cAAcT;gCACpBvB,MAAM,2BAA2BgC;gCAGjC,MAAM,CAACP,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvB6B,YAAY,KAAK,CAAC,EAAE,EACpBA,YAAY,KAAK,CAAC,EAAE,EACpBX,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAGA,MAAMK,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWE,KAAK,KAAK,CAClBC,YAAYlB,KAAK,MAAM,GAAInB;oCAE9BiC,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWE,KAAK,KAAK,CAClBD,YAAYhB,KAAK,KAAK,GAAInB;oCAE7BiC,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEAjC,MACE,CAAC,2BAA2B,EAAEmC,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLP;4CAEAO;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMQ,kBAAkBjB;gCACxBvB,MAAM,gCAAgCwC;gCACtC,MAAM,CAACf,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvBqC,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BnB,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;wCACA,SAASW,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAalB;gCACnBvB,MAAM,0BAA0ByC;gCAChC,OAAO;oCACL;wCACE,MAAM5B,eACJC,aACAH,mBACA;wCAEF,OAAO,CAAC;wCACR,SAAS8B,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAanB;gCACnBvB,MAAM,0BAA0B0C;gCAChC,OAAO;oCACL;wCACE,MAAM7B,eACJC,aACAF,mBACA;wCAEF,OAAO,CAAC;wCACR,SAAS8B,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAapB;gCACnBvB,MAAM,0BAA0B2C;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAerB;gCACrBvB,MAAM,4BAA4B4C;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CAAE,KAAKA,aAAa,GAAG;wCAAC;wCAC/B,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGtB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIsB,MACR,CAAC,yBAAyB,EAAGzB,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAO0B,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrE9C,MAAM,oBAAoB+C;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}

package/dist/lib/ai-model/auto-glm/index.js

@@ -13,7 +13,7 @@ var __webpack_require__ = {};
 })();
 (()=>{
     __webpack_require__.r = (exports1)=>{
-        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
             value: 'Module'
         });
         Object.defineProperty(exports1, '__esModule', {

package/dist/lib/ai-model/auto-glm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/auto-glm/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,MAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}
+{"version":3,"file":"ai-model/auto-glm/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/lib/ai-model/auto-glm/parser.js

@@ -13,7 +13,7 @@ var __webpack_require__ = {};
 })();
 (()=>{
     __webpack_require__.r = (exports1)=>{
-        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
             value: 'Module'
         });
         Object.defineProperty(exports1, '__esModule', {

package/dist/lib/ai-model/auto-glm/parser.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/auto-glm/parser.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/parser.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getDebug } from '@aiscene/shared/logger';\nimport type { ParsedAction } from './actions';\n\nconst debug = getDebug('auto-glm-parser');\n\n// Do not rely on regex extraction here; regex can fail on malformed input.\n// Bad Case: finish(message=\"Finished! Now There is a contact whose name is \\\"Tom\\\" in the list.\")\nexport const extractValueAfter = (src: string, key: string): string => {\n  const idx = src.indexOf(key);\n  if (idx === -1) {\n    throw new Error(`Missing key ${key} in action payload ${src}`);\n  }\n  let rest = src.slice(idx + key.length).trim();\n  if (rest.endsWith('\")')) {\n    rest = rest.slice(0, -2);\n  }\n  return rest;\n};\n\nexport function parseAction(response: {\n  think: string;\n  content: string;\n}): ParsedAction {\n  debug('Parsing action:', response);\n  let trimmedResponse = '';\n  try {\n    trimmedResponse = response.content.trim();\n\n    if (\n      trimmedResponse.startsWith('do(action=\"Type\"') ||\n      trimmedResponse.startsWith('do(action=\"Type_Name\"')\n    ) {\n      const text = extractValueAfter(trimmedResponse, 'text=\"');\n      return {\n        _metadata: 'do',\n        action: 'Type',\n        text,\n        think: response.think,\n      } as ParsedAction;\n    }\n\n    if (trimmedResponse.startsWith('finish(message=')) {\n      let message = extractValueAfter(trimmedResponse, 'finish(message=\"');\n      if (message.endsWith(')')) message = message.slice(0, -1);\n      return {\n        _metadata: 'finish',\n        message,\n        think: response.think,\n      } as ParsedAction;\n    }\n\n    if (trimmedResponse.startsWith('do(')) {\n      const actionMatch = trimmedResponse.match(/do\\(action=\"([^\"]+)\"/);\n      if (!actionMatch)\n        throw new Error(\n          `Failed to extract action type from do() call; raw=\"${trimmedResponse}\"`,\n        );\n      const actionType = actionMatch[1];\n\n      const baseAction = { _metadata: 'do' as const, think: response.think };\n      switch (actionType) {\n        case 'Tap': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Tap; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Tap',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Double Tap': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Double Tap; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Double Tap',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Swipe': {\n          const startMatch = trimmedResponse.match(/start=\\[(\\d+),(\\d+)\\]/);\n          const endMatch = trimmedResponse.match(/end=\\[(\\d+),(\\d+)\\]/);\n          if (!startMatch || !endMatch)\n            throw new Error(\n              `Failed to extract start/end coordinates for Swipe; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Swipe',\n            start: [Number(startMatch[1]), Number(startMatch[2])],\n            end: [Number(endMatch[1]), Number(endMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Long Press': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Long Press; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Long Press',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Launch': {\n          const app = extractValueAfter(trimmedResponse, 'app=\"');\n          return { ...baseAction, action: 'Launch', app } as ParsedAction;\n        }\n        case 'Back': {\n          return { ...baseAction, action: 'Back' } as ParsedAction;\n        }\n        case 'Home': {\n          return { ...baseAction, action: 'Home' } as ParsedAction;\n        }\n        case 'Wait': {\n          const durationMatch = trimmedResponse.match(\n            /duration=(?:[\"\\[])?(\\d+)/,\n          );\n          if (!durationMatch) {\n            throw new Error(\n              `Failed to extract duration for Wait; raw=\"${trimmedResponse}\"`,\n            );\n          }\n          const seconds = Number.parseInt(durationMatch[1], 10);\n          const durationMs = seconds * 1000;\n          return {\n            ...baseAction,\n            action: 'Wait',\n            durationMs,\n          } as ParsedAction;\n        }\n        case 'Interact': {\n          return { ...baseAction, action: 'Interact' } as ParsedAction;\n        }\n        case 'Call_API': {\n          const instruction = extractValueAfter(\n            trimmedResponse,\n            'instruction=\"',\n          );\n          return {\n            ...baseAction,\n            action: 'Call_API',\n            instruction,\n          } as ParsedAction;\n        }\n        case 'Take_over': {\n          const message = extractValueAfter(trimmedResponse, 'message=\"');\n          return {\n            ...baseAction,\n            action: 'Take_over',\n            message,\n          } as ParsedAction;\n        }\n        case 'Note': {\n          const message = extractValueAfter(trimmedResponse, 'message=\"');\n          return {\n            ...baseAction,\n            action: 'Note',\n            message,\n          } as ParsedAction;\n        }\n        default:\n          throw new Error(\n            `Unknown action type: ${actionType}; raw=\"${trimmedResponse}\"`,\n          );\n      }\n    }\n    throw new Error(`Failed to parse action: ${trimmedResponse}`);\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    throw new Error(\n      `Failed to parse action: ${errorMessage}; raw=\"${trimmedResponse}\"`,\n    );\n  }\n}\n\nexport function parseAutoGLMResponse(content: string): {\n  think: string;\n  content: string;\n} {\n  if (content.includes('finish(message=')) {\n    const parts = content.split('finish(message=');\n    const think = parts[0].trim();\n    const actionContent = `finish(message=${parts[1]}`;\n    return { think, content: actionContent };\n  }\n  if (content.includes('do(action=')) {\n    const parts = content.split('do(action=');\n    const think = parts[0].trim();\n    const actionContent = `do(action=${parts[1]}`;\n    return { think, content: actionContent };\n  }\n  if (content.includes('<answer>')) {\n    const parts = content.split('<answer>');\n    const think = parts[0]\n      .replace(/<think>/g, '')\n      .replace(/<\\/think>/g, '')\n      .trim();\n    const actionContent = parts[1].replace(/<\\/answer>/g, '').trim();\n    return { think, content: actionContent };\n  }\n  return { think: '', content };\n}\n\nexport function parseAutoGLMLocateResponse(rawResponse: string): {\n  think: string;\n  coordinates: { x: number; y: number } | null;\n  error?: string;\n} {\n  const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);\n  if (!actionContent.startsWith('do(action=\"Tap\"')) {\n    return {\n      think,\n      coordinates: null,\n      error: `Unexpected action type in auto-glm locate response: ${actionContent}`,\n    };\n  }\n  try {\n    const elementMatch = actionContent.match(/element=\\[(\\d+),(\\d+)\\]/);\n    if (!elementMatch) {\n      return {\n        think,\n        coordinates: null,\n        error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`,\n      };\n    }\n    const x = Number(elementMatch[1]);\n    const y = Number(elementMatch[2]);\n    return { think, coordinates: { x, y } };\n  } catch (e) {\n    const errorMessage = e instanceof Error ? e.message : String(e);\n    return {\n      think,\n      coordinates: null,\n      error: `Failed to parse coordinates \"${actionContent}\" with errorMessage: ${errorMessage}`,\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","extractValueAfter","src","idx","Error","rest","parseAction","response","trimmedResponse","text","message","actionMatch","actionType","baseAction","elementMatch","Number","startMatch","endMatch","app","durationMatch","seconds","durationMs","instruction","error","errorMessage","String","parseAutoGLMResponse","content","parts","think","actionContent","parseAutoGLMLocateResponse","rawResponse","x","y","e"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,MAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACHA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIhB,MAAMC,oBAAoB,CAACC,KAAaR;IAC7C,MAAMS,MAAMD,IAAI,OAAO,CAACR;IACxB,IAAIS,AAAQ,OAARA,KACF,MAAM,IAAIC,MAAM,CAAC,YAAY,EAAEV,IAAI,mBAAmB,EAAEQ,KAAK;IAE/D,IAAIG,OAAOH,IAAI,KAAK,CAACC,MAAMT,IAAI,MAAM,EAAE,IAAI;IAC3C,IAAIW,KAAK,QAAQ,CAAC,OAChBA,OAAOA,KAAK,KAAK,CAAC,GAAG;IAEvB,OAAOA;AACT;AAEO,SAASC,YAAYC,QAG3B;IACCR,MAAM,mBAAmBQ;IACzB,IAAIC,kBAAkB;IACtB,IAAI;QACFA,kBAAkBD,SAAS,OAAO,CAAC,IAAI;QAEvC,IACEC,gBAAgB,UAAU,CAAC,uBAC3BA,gBAAgB,UAAU,CAAC,0BAC3B;YACA,MAAMC,OAAOR,kBAAkBO,iBAAiB;YAChD,OAAO;gBACL,WAAW;gBACX,QAAQ;gBACRC;gBACA,OAAOF,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,oBAAoB;YACjD,IAAIE,UAAUT,kBAAkBO,iBAAiB;YACjD,IAAIE,QAAQ,QAAQ,CAAC,MAAMA,UAAUA,QAAQ,KAAK,CAAC,GAAG;YACtD,OAAO;gBACL,WAAW;gBACXA;gBACA,OAAOH,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,QAAQ;YACrC,MAAMG,cAAcH,gBAAgB,KAAK,CAAC;YAC1C,IAAI,CAACG,aACH,MAAM,IAAIP,MACR,CAAC,mDAAmD,EAAEI,gBAAgB,CAAC,CAAC;YAE5E,MAAMI,aAAaD,WAAW,CAAC,EAAE;YAEjC,MAAME,aAAa;gBAAE,WAAW;gBAAe,OAAON,SAAS,KAAK;YAAC;YACrE,OAAQK;gBACN,KAAK;oBAAO;wBACV,MAAME,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,oDAAoD,EAAEI,gBAAgB,CAAC,CAAC;wBAE7E,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMA,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAS;wBACZ,MAAME,aAAaR,gBAAgB,KAAK,CAAC;wBACzC,MAAMS,WAAWT,gBAAgB,KAAK,CAAC;wBACvC,IAAI,CAACQ,cAAc,CAACC,UAClB,MAAM,IAAIb,MACR,CAAC,wDAAwD,EAAEI,gBAAgB,CAAC,CAAC;wBAEjF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,OAAO;gCAACE,OAAOC,UAAU,CAAC,EAAE;gCAAGD,OAAOC,UAAU,CAAC,EAAE;6BAAE;4BACrD,KAAK;gCAACD,OAAOE,QAAQ,CAAC,EAAE;gCAAGF,OAAOE,QAAQ,CAAC,EAAE;6BAAE;wBACjD;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMH,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAU;wBACb,MAAMI,MAAMjB,kBAAkBO,iBAAiB;wBAC/C,OAAO;4BAAE,GAAGK,UAAU;4BAAE,QAAQ;4BAAUK;wBAAI;oBAChD;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGL,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBACH,OAAO;wBAAE,GAAGA,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBAAQ;wBACX,MAAMM,gBAAgBX,gBAAgB,KAAK,CACzC;wBAEF,IAAI,CAACW,eACH,MAAM,IAAIf,MACR,CAAC,0CAA0C,EAAEI,gBAAgB,CAAC,CAAC;wBAGnE,MAAMY,UAAUL,OAAO,QAAQ,CAACI,aAAa,CAAC,EAAE,EAAE;wBAClD,MAAME,aAAaD,AAAU,OAAVA;wBACnB,OAAO;4BACL,GAAGP,UAAU;4BACb,QAAQ;4BACRQ;wBACF;oBACF;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGR,UAAU;wBAAE,QAAQ;oBAAW;gBAE7C,KAAK;oBAAY;wBACf,MAAMS,cAAcrB,kBAClBO,iBACA;wBAEF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRS;wBACF;oBACF;gBACA,KAAK;oBAAa;wBAChB,MAAMZ,UAAUT,kBAAkBO,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA,KAAK;oBAAQ;wBACX,MAAMA,UAAUT,kBAAkBO,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA;oBACE,MAAM,IAAIN,MACR,CAAC,qBAAqB,EAAEQ,WAAW,OAAO,EAAEJ,gBAAgB,CAAC,CAAC;YAEpE;QACF;QACA,MAAM,IAAIJ,MAAM,CAAC,wBAAwB,EAAEI,iBAAiB;IAC9D,EAAE,OAAOe,OAAO;QACd,MAAMC,eAAeD,iBAAiBnB,QAAQmB,MAAM,OAAO,GAAGE,OAAOF;QACrE,MAAM,IAAInB,MACR,CAAC,wBAAwB,EAAEoB,aAAa,OAAO,EAAEhB,gBAAgB,CAAC,CAAC;IAEvE;AACF;AAEO,SAASkB,qBAAqBC,OAAe;IAIlD,IAAIA,QAAQ,QAAQ,CAAC,oBAAoB;QACvC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,eAAe,EAAEF,KAAK,CAAC,EAAE,EAAE;QAClD,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,IAAIH,QAAQ,QAAQ,CAAC,eAAe;QAClC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,UAAU,EAAEF,KAAK,CAAC,EAAE,EAAE;QAC7C,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,IAAIH,QAAQ,QAAQ,CAAC,aAAa;QAChC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CACnB,OAAO,CAAC,YAAY,IACpB,OAAO,CAAC,cAAc,IACtB,IAAI;QACP,MAAME,gBAAgBF,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,IAAI;QAC9D,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,OAAO;QAAE,OAAO;QAAIH;IAAQ;AAC9B;AAEO,SAASI,2BAA2BC,WAAmB;IAK5D,MAAM,EAAEH,KAAK,EAAE,SAASC,aAAa,EAAE,GAAGJ,qBAAqBM;IAC/D,IAAI,CAACF,cAAc,UAAU,CAAC,oBAC5B,OAAO;QACLD;QACA,aAAa;QACb,OAAO,CAAC,oDAAoD,EAAEC,eAAe;IAC/E;IAEF,IAAI;QACF,MAAMhB,eAAegB,cAAc,KAAK,CAAC;QACzC,IAAI,CAAChB,cACH,OAAO;YACLe;YACA,aAAa;YACb,OAAO,CAAC,8DAA8D,EAAEC,eAAe;QACzF;QAEF,MAAMG,IAAIlB,OAAOD,YAAY,CAAC,EAAE;QAChC,MAAMoB,IAAInB,OAAOD,YAAY,CAAC,EAAE;QAChC,OAAO;YAAEe;YAAO,aAAa;gBAAEI;gBAAGC;YAAE;QAAE;IACxC,EAAE,OAAOC,GAAG;QACV,MAAMX,eAAeW,aAAa/B,QAAQ+B,EAAE,OAAO,GAAGV,OAAOU;QAC7D,OAAO;YACLN;YACA,aAAa;YACb,OAAO,CAAC,6BAA6B,EAAEC,cAAc,qBAAqB,EAAEN,cAAc;QAC5F;IACF;AACF"}
+{"version":3,"file":"ai-model/auto-glm/parser.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/parser.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getDebug } from '@aiscene/shared/logger';\nimport type { ParsedAction } from './actions';\n\nconst debug = getDebug('auto-glm-parser');\n\n// Do not rely on regex extraction here; regex can fail on malformed input.\n// Bad Case: finish(message=\"Finished! Now There is a contact whose name is \\\"Tom\\\" in the list.\")\nexport const extractValueAfter = (src: string, key: string): string => {\n  const idx = src.indexOf(key);\n  if (idx === -1) {\n    throw new Error(`Missing key ${key} in action payload ${src}`);\n  }\n  let rest = src.slice(idx + key.length).trim();\n  if (rest.endsWith('\")')) {\n    rest = rest.slice(0, -2);\n  }\n  return rest;\n};\n\nexport function parseAction(response: {\n  think: string;\n  content: string;\n}): ParsedAction {\n  debug('Parsing action:', response);\n  let trimmedResponse = '';\n  try {\n    trimmedResponse = response.content.trim();\n\n    if (\n      trimmedResponse.startsWith('do(action=\"Type\"') ||\n      trimmedResponse.startsWith('do(action=\"Type_Name\"')\n    ) {\n      const text = extractValueAfter(trimmedResponse, 'text=\"');\n      return {\n        _metadata: 'do',\n        action: 'Type',\n        text,\n        think: response.think,\n      } as ParsedAction;\n    }\n\n    if (trimmedResponse.startsWith('finish(message=')) {\n      let message = extractValueAfter(trimmedResponse, 'finish(message=\"');\n      if (message.endsWith(')')) message = message.slice(0, -1);\n      return {\n        _metadata: 'finish',\n        message,\n        think: response.think,\n      } as ParsedAction;\n    }\n\n    if (trimmedResponse.startsWith('do(')) {\n      const actionMatch = trimmedResponse.match(/do\\(action=\"([^\"]+)\"/);\n      if (!actionMatch)\n        throw new Error(\n          `Failed to extract action type from do() call; raw=\"${trimmedResponse}\"`,\n        );\n      const actionType = actionMatch[1];\n\n      const baseAction = { _metadata: 'do' as const, think: response.think };\n      switch (actionType) {\n        case 'Tap': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Tap; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Tap',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Double Tap': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Double Tap; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Double Tap',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Swipe': {\n          const startMatch = trimmedResponse.match(/start=\\[(\\d+),(\\d+)\\]/);\n          const endMatch = trimmedResponse.match(/end=\\[(\\d+),(\\d+)\\]/);\n          if (!startMatch || !endMatch)\n            throw new Error(\n              `Failed to extract start/end coordinates for Swipe; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Swipe',\n            start: [Number(startMatch[1]), Number(startMatch[2])],\n            end: [Number(endMatch[1]), Number(endMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Long Press': {\n          const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n          if (!elementMatch)\n            throw new Error(\n              `Failed to extract element coordinates for Long Press; raw=\"${trimmedResponse}\"`,\n            );\n          return {\n            ...baseAction,\n            action: 'Long Press',\n            element: [Number(elementMatch[1]), Number(elementMatch[2])],\n          } as ParsedAction;\n        }\n        case 'Launch': {\n          const app = extractValueAfter(trimmedResponse, 'app=\"');\n          return { ...baseAction, action: 'Launch', app } as ParsedAction;\n        }\n        case 'Back': {\n          return { ...baseAction, action: 'Back' } as ParsedAction;\n        }\n        case 'Home': {\n          return { ...baseAction, action: 'Home' } as ParsedAction;\n        }\n        case 'Wait': {\n          const durationMatch = trimmedResponse.match(\n            /duration=(?:[\"\\[])?(\\d+)/,\n          );\n          if (!durationMatch) {\n            throw new Error(\n              `Failed to extract duration for Wait; raw=\"${trimmedResponse}\"`,\n            );\n          }\n          const seconds = Number.parseInt(durationMatch[1], 10);\n          const durationMs = seconds * 1000;\n          return {\n            ...baseAction,\n            action: 'Wait',\n            durationMs,\n          } as ParsedAction;\n        }\n        case 'Interact': {\n          return { ...baseAction, action: 'Interact' } as ParsedAction;\n        }\n        case 'Call_API': {\n          const instruction = extractValueAfter(\n            trimmedResponse,\n            'instruction=\"',\n          );\n          return {\n            ...baseAction,\n            action: 'Call_API',\n            instruction,\n          } as ParsedAction;\n        }\n        case 'Take_over': {\n          const message = extractValueAfter(trimmedResponse, 'message=\"');\n          return {\n            ...baseAction,\n            action: 'Take_over',\n            message,\n          } as ParsedAction;\n        }\n        case 'Note': {\n          const message = extractValueAfter(trimmedResponse, 'message=\"');\n          return {\n            ...baseAction,\n            action: 'Note',\n            message,\n          } as ParsedAction;\n        }\n        default:\n          throw new Error(\n            `Unknown action type: ${actionType}; raw=\"${trimmedResponse}\"`,\n          );\n      }\n    }\n    throw new Error(`Failed to parse action: ${trimmedResponse}`);\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    throw new Error(\n      `Failed to parse action: ${errorMessage}; raw=\"${trimmedResponse}\"`,\n    );\n  }\n}\n\nexport function parseAutoGLMResponse(content: string): {\n  think: string;\n  content: string;\n} {\n  if (content.includes('finish(message=')) {\n    const parts = content.split('finish(message=');\n    const think = parts[0].trim();\n    const actionContent = `finish(message=${parts[1]}`;\n    return { think, content: actionContent };\n  }\n  if (content.includes('do(action=')) {\n    const parts = content.split('do(action=');\n    const think = parts[0].trim();\n    const actionContent = `do(action=${parts[1]}`;\n    return { think, content: actionContent };\n  }\n  if (content.includes('<answer>')) {\n    const parts = content.split('<answer>');\n    const think = parts[0]\n      .replace(/<think>/g, '')\n      .replace(/<\\/think>/g, '')\n      .trim();\n    const actionContent = parts[1].replace(/<\\/answer>/g, '').trim();\n    return { think, content: actionContent };\n  }\n  return { think: '', content };\n}\n\nexport function parseAutoGLMLocateResponse(rawResponse: string): {\n  think: string;\n  coordinates: { x: number; y: number } | null;\n  error?: string;\n} {\n  const { think, content: actionContent } = parseAutoGLMResponse(rawResponse);\n  if (!actionContent.startsWith('do(action=\"Tap\"')) {\n    return {\n      think,\n      coordinates: null,\n      error: `Unexpected action type in auto-glm locate response: ${actionContent}`,\n    };\n  }\n  try {\n    const elementMatch = actionContent.match(/element=\\[(\\d+),(\\d+)\\]/);\n    if (!elementMatch) {\n      return {\n        think,\n        coordinates: null,\n        error: `Failed to extract element coordinates from auto-glm response: ${actionContent}`,\n      };\n    }\n    const x = Number(elementMatch[1]);\n    const y = Number(elementMatch[2]);\n    return { think, coordinates: { x, y } };\n  } catch (e) {\n    const errorMessage = e instanceof Error ? e.message : String(e);\n    return {\n      think,\n      coordinates: null,\n      error: `Failed to parse coordinates \"${actionContent}\" with errorMessage: ${errorMessage}`,\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","extractValueAfter","src","idx","Error","rest","parseAction","response","trimmedResponse","text","message","actionMatch","actionType","baseAction","elementMatch","Number","startMatch","endMatch","app","durationMatch","seconds","durationMs","instruction","error","errorMessage","String","parseAutoGLMResponse","content","parts","think","actionContent","parseAutoGLMLocateResponse","rawResponse","x","y","e"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACHA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIhB,MAAMC,oBAAoB,CAACC,KAAaR;IAC7C,MAAMS,MAAMD,IAAI,OAAO,CAACR;IACxB,IAAIS,AAAQ,OAARA,KACF,MAAM,IAAIC,MAAM,CAAC,YAAY,EAAEV,IAAI,mBAAmB,EAAEQ,KAAK;IAE/D,IAAIG,OAAOH,IAAI,KAAK,CAACC,MAAMT,IAAI,MAAM,EAAE,IAAI;IAC3C,IAAIW,KAAK,QAAQ,CAAC,OAChBA,OAAOA,KAAK,KAAK,CAAC,GAAG;IAEvB,OAAOA;AACT;AAEO,SAASC,YAAYC,QAG3B;IACCR,MAAM,mBAAmBQ;IACzB,IAAIC,kBAAkB;IACtB,IAAI;QACFA,kBAAkBD,SAAS,OAAO,CAAC,IAAI;QAEvC,IACEC,gBAAgB,UAAU,CAAC,uBAC3BA,gBAAgB,UAAU,CAAC,0BAC3B;YACA,MAAMC,OAAOR,kBAAkBO,iBAAiB;YAChD,OAAO;gBACL,WAAW;gBACX,QAAQ;gBACRC;gBACA,OAAOF,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,oBAAoB;YACjD,IAAIE,UAAUT,kBAAkBO,iBAAiB;YACjD,IAAIE,QAAQ,QAAQ,CAAC,MAAMA,UAAUA,QAAQ,KAAK,CAAC,GAAG;YACtD,OAAO;gBACL,WAAW;gBACXA;gBACA,OAAOH,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,QAAQ;YACrC,MAAMG,cAAcH,gBAAgB,KAAK,CAAC;YAC1C,IAAI,CAACG,aACH,MAAM,IAAIP,MACR,CAAC,mDAAmD,EAAEI,gBAAgB,CAAC,CAAC;YAE5E,MAAMI,aAAaD,WAAW,CAAC,EAAE;YAEjC,MAAME,aAAa;gBAAE,WAAW;gBAAe,OAAON,SAAS,KAAK;YAAC;YACrE,OAAQK;gBACN,KAAK;oBAAO;wBACV,MAAME,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,oDAAoD,EAAEI,gBAAgB,CAAC,CAAC;wBAE7E,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMA,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAS;wBACZ,MAAME,aAAaR,gBAAgB,KAAK,CAAC;wBACzC,MAAMS,WAAWT,gBAAgB,KAAK,CAAC;wBACvC,IAAI,CAACQ,cAAc,CAACC,UAClB,MAAM,IAAIb,MACR,CAAC,wDAAwD,EAAEI,gBAAgB,CAAC,CAAC;wBAEjF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,OAAO;gCAACE,OAAOC,UAAU,CAAC,EAAE;gCAAGD,OAAOC,UAAU,CAAC,EAAE;6BAAE;4BACrD,KAAK;gCAACD,OAAOE,QAAQ,CAAC,EAAE;gCAAGF,OAAOE,QAAQ,CAAC,EAAE;6BAAE;wBACjD;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMH,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAU;wBACb,MAAMI,MAAMjB,kBAAkBO,iBAAiB;wBAC/C,OAAO;4BAAE,GAAGK,UAAU;4BAAE,QAAQ;4BAAUK;wBAAI;oBAChD;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGL,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBACH,OAAO;wBAAE,GAAGA,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBAAQ;wBACX,MAAMM,gBAAgBX,gBAAgB,KAAK,CACzC;wBAEF,IAAI,CAACW,eACH,MAAM,IAAIf,MACR,CAAC,0CAA0C,EAAEI,gBAAgB,CAAC,CAAC;wBAGnE,MAAMY,UAAUL,OAAO,QAAQ,CAACI,aAAa,CAAC,EAAE,EAAE;wBAClD,MAAME,aAAaD,AAAU,OAAVA;wBACnB,OAAO;4BACL,GAAGP,UAAU;4BACb,QAAQ;4BACRQ;wBACF;oBACF;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGR,UAAU;wBAAE,QAAQ;oBAAW;gBAE7C,KAAK;oBAAY;wBACf,MAAMS,cAAcrB,kBAClBO,iBACA;wBAEF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRS;wBACF;oBACF;gBACA,KAAK;oBAAa;wBAChB,MAAMZ,UAAUT,kBAAkBO,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA,KAAK;oBAAQ;wBACX,MAAMA,UAAUT,kBAAkBO,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA;oBACE,MAAM,IAAIN,MACR,CAAC,qBAAqB,EAAEQ,WAAW,OAAO,EAAEJ,gBAAgB,CAAC,CAAC;YAEpE;QACF;QACA,MAAM,IAAIJ,MAAM,CAAC,wBAAwB,EAAEI,iBAAiB;IAC9D,EAAE,OAAOe,OAAO;QACd,MAAMC,eAAeD,iBAAiBnB,QAAQmB,MAAM,OAAO,GAAGE,OAAOF;QACrE,MAAM,IAAInB,MACR,CAAC,wBAAwB,EAAEoB,aAAa,OAAO,EAAEhB,gBAAgB,CAAC,CAAC;IAEvE;AACF;AAEO,SAASkB,qBAAqBC,OAAe;IAIlD,IAAIA,QAAQ,QAAQ,CAAC,oBAAoB;QACvC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,eAAe,EAAEF,KAAK,CAAC,EAAE,EAAE;QAClD,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,IAAIH,QAAQ,QAAQ,CAAC,eAAe;QAClC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,UAAU,EAAEF,KAAK,CAAC,EAAE,EAAE;QAC7C,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,IAAIH,QAAQ,QAAQ,CAAC,aAAa;QAChC,MAAMC,QAAQD,QAAQ,KAAK,CAAC;QAC5B,MAAME,QAAQD,KAAK,CAAC,EAAE,CACnB,OAAO,CAAC,YAAY,IACpB,OAAO,CAAC,cAAc,IACtB,IAAI;QACP,MAAME,gBAAgBF,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,IAAI;QAC9D,OAAO;YAAEC;YAAO,SAASC;QAAc;IACzC;IACA,OAAO;QAAE,OAAO;QAAIH;IAAQ;AAC9B;AAEO,SAASI,2BAA2BC,WAAmB;IAK5D,MAAM,EAAEH,KAAK,EAAE,SAASC,aAAa,EAAE,GAAGJ,qBAAqBM;IAC/D,IAAI,CAACF,cAAc,UAAU,CAAC,oBAC5B,OAAO;QACLD;QACA,aAAa;QACb,OAAO,CAAC,oDAAoD,EAAEC,eAAe;IAC/E;IAEF,IAAI;QACF,MAAMhB,eAAegB,cAAc,KAAK,CAAC;QACzC,IAAI,CAAChB,cACH,OAAO;YACLe;YACA,aAAa;YACb,OAAO,CAAC,8DAA8D,EAAEC,eAAe;QACzF;QAEF,MAAMG,IAAIlB,OAAOD,YAAY,CAAC,EAAE;QAChC,MAAMoB,IAAInB,OAAOD,YAAY,CAAC,EAAE;QAChC,OAAO;YAAEe;YAAO,aAAa;gBAAEI;gBAAGC;YAAE;QAAE;IACxC,EAAE,OAAOC,GAAG;QACV,MAAMX,eAAeW,aAAa/B,QAAQ+B,EAAE,OAAO,GAAGV,OAAOU;QAC7D,OAAO;YACLN;YACA,aAAa;YACb,OAAO,CAAC,6BAA6B,EAAEC,cAAc,qBAAqB,EAAEN,cAAc;QAC5F;IACF;AACF"}

package/dist/lib/ai-model/auto-glm/planning.js

@@ -13,7 +13,7 @@ var __webpack_require__ = {};
 })();
 (()=>{
     __webpack_require__.r = (exports1)=>{
-        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
             value: 'Module'
         });
         Object.defineProperty(exports1, '__esModule', {
@@ -36,12 +36,13 @@ async function autoGLMPlanning(userInstruction, options) {
     const { conversationHistory, context, modelConfig, actionContext } = options;
     const systemPrompt = (0, external_prompt_js_namespaceObject.getAutoGLMPlanPrompt)(modelConfig.modelFamily) + (actionContext ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>` : '');
     const imagePayloadBase64 = context.screenshot.base64;
+    const memoriesText = conversationHistory.memoriesToText();
     conversationHistory.append({
         role: 'user',
         content: [
             {
                 type: 'text',
-                text: userInstruction
+                text: memoriesText ? `${memoriesText}\n${userInstruction}` : userInstruction
             }
         ]
     });

package/dist/lib/ai-model/auto-glm/planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/auto-glm/planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/device';\nimport type { PlanningAIResponse, UIContext } from '@/types';\nimport type { IModelConfig } from '@aiscene/shared/env';\nimport { getDebug } from '@aiscene/shared/logger';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { ConversationHistory } from '../conversation-history';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../service-caller/index';\nimport { transformAutoGLMAction } from './actions';\nimport { parseAction, parseAutoGLMResponse } from './parser';\nimport { getAutoGLMPlanPrompt } from './prompt';\n\nconst debug = getDebug('auto-glm-planning');\n\nexport async function autoGLMPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n    actionContext?: string;\n    actionSpace?: DeviceAction[];\n    abortSignal?: AbortSignal;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig, actionContext } = options;\n\n  const systemPrompt =\n    getAutoGLMPlanPrompt(modelConfig.modelFamily) +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>`\n      : '');\n\n  const imagePayloadBase64 = context.screenshot.base64;\n\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'text', text: userInstruction }],\n  });\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'image_url', image_url: { url: imagePayloadBase64 } }],\n  });\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...conversationHistory.snapshot(1),\n  ];\n\n  const { content: rawResponse, usage } = await callAIWithStringResponse(\n    msgs,\n    modelConfig,\n    {\n      abortSignal: options.abortSignal,\n    },\n  );\n\n  debug('autoGLMPlanning rawResponse:', rawResponse);\n\n  let parsedResponse: ReturnType<typeof parseAutoGLMResponse>;\n  let transformedActions: ReturnType<typeof transformAutoGLMAction>;\n\n  try {\n    parsedResponse = parseAutoGLMResponse(rawResponse);\n    debug('thinking in response:', parsedResponse.think);\n    debug('action in response:', parsedResponse.content);\n\n    const parsedAction = parseAction(parsedResponse);\n    debug('Parsed action object:', parsedAction);\n    transformedActions = transformAutoGLMAction(\n      parsedAction,\n      context.shotSize,\n      options.actionSpace,\n    );\n    debug('Transformed actions:', transformedActions);\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `Parse error: ${errorMessage}`,\n      JSON.stringify(rawResponse, undefined, 2),\n      usage,\n    );\n  }\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: `<think>${parsedResponse.think}</think><answer>${parsedResponse.content}</answer>`,\n  });\n\n  const shouldContinuePlanning = !parsedResponse.content.startsWith('finish(');\n\n  return {\n    actions: transformedActions,\n    log: rawResponse,\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(rawResponse, undefined, 2),\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","autoGLMPlanning","userInstruction","options","conversationHistory","context","modelConfig","actionContext","systemPrompt","getAutoGLMPlanPrompt","imagePayloadBase64","msgs","rawResponse","usage","callAIWithStringResponse","parsedResponse","transformedActions","parseAutoGLMResponse","parsedAction","parseAction","transformAutoGLMAction","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","shouldContinuePlanning"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,MAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACQA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,gBACpBC,eAAuB,EACvBC,OAOC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAEC,aAAa,EAAE,GAAGJ;IAErE,MAAMK,eACJC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBH,YAAY,WAAW,IAC3CC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,0BAA0B,CAAC,GACrE,EAAC;IAEP,MAAMG,qBAAqBL,QAAQ,UAAU,CAAC,MAAM;IAEpDD,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAQ,MAAMF;YAAgB;SAAE;IACpD;IACAE,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAa,WAAW;oBAAE,KAAKM;gBAAmB;YAAE;SAAE;IAC1E;IAEA,MAAMC,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASH;QAAa;WACrCJ,oBAAoB,QAAQ,CAAC;KACjC;IAED,MAAM,EAAE,SAASQ,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAC5CH,MACAL,aACA;QACE,aAAaH,QAAQ,WAAW;IAClC;IAGFJ,MAAM,gCAAgCa;IAEtC,IAAIG;IACJ,IAAIC;IAEJ,IAAI;QACFD,iBAAiBE,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBL;QACtCb,MAAM,yBAAyBgB,eAAe,KAAK;QACnDhB,MAAM,uBAAuBgB,eAAe,OAAO;QAEnD,MAAMG,eAAeC,AAAAA,IAAAA,mCAAAA,WAAAA,AAAAA,EAAYJ;QACjChB,MAAM,yBAAyBmB;QAC/BF,qBAAqBI,AAAAA,IAAAA,oCAAAA,sBAAAA,AAAAA,EACnBF,cACAb,QAAQ,QAAQ,EAChBF,QAAQ,WAAW;QAErBJ,MAAM,wBAAwBiB;IAChC,EAAE,OAAOK,YAAY;QAEnB,MAAMC,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;QAC5D,MAAM,IAAII,yBAAAA,oBAAoBA,CAC5B,CAAC,aAAa,EAAEH,cAAc,EAC9BI,KAAK,SAAS,CAACd,aAAae,QAAW,IACvCd;IAEJ;IAEAT,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS,CAAC,OAAO,EAAEW,eAAe,KAAK,CAAC,gBAAgB,EAAEA,eAAe,OAAO,CAAC,SAAS,CAAC;IAC7F;IAEA,MAAMa,yBAAyB,CAACb,eAAe,OAAO,CAAC,UAAU,CAAC;IAElE,OAAO;QACL,SAASC;QACT,KAAKJ;QACLC;QACAe;QACA,aAAaF,KAAK,SAAS,CAACd,aAAae,QAAW;IACtD;AACF"}
+{"version":3,"file":"ai-model/auto-glm/planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/auto-glm/planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/device';\nimport type { PlanningAIResponse, UIContext } from '@/types';\nimport type { IModelConfig } from '@aiscene/shared/env';\nimport { getDebug } from '@aiscene/shared/logger';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { ConversationHistory } from '../conversation-history';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../service-caller/index';\nimport { transformAutoGLMAction } from './actions';\nimport { parseAction, parseAutoGLMResponse } from './parser';\nimport { getAutoGLMPlanPrompt } from './prompt';\n\nconst debug = getDebug('auto-glm-planning');\n\nexport async function autoGLMPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n    actionContext?: string;\n    actionSpace?: DeviceAction[];\n    abortSignal?: AbortSignal;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig, actionContext } = options;\n\n  const systemPrompt =\n    getAutoGLMPlanPrompt(modelConfig.modelFamily) +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>`\n      : '');\n\n  const imagePayloadBase64 = context.screenshot.base64;\n  const memoriesText = conversationHistory.memoriesToText();\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'text',\n        text: memoriesText\n          ? `${memoriesText}\\n${userInstruction}`\n          : userInstruction,\n      },\n    ],\n  });\n  conversationHistory.append({\n    role: 'user',\n    content: [{ type: 'image_url', image_url: { url: imagePayloadBase64 } }],\n  });\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...conversationHistory.snapshot(1),\n  ];\n\n  const { content: rawResponse, usage } = await callAIWithStringResponse(\n    msgs,\n    modelConfig,\n    {\n      abortSignal: options.abortSignal,\n    },\n  );\n\n  debug('autoGLMPlanning rawResponse:', rawResponse);\n\n  let parsedResponse: ReturnType<typeof parseAutoGLMResponse>;\n  let transformedActions: ReturnType<typeof transformAutoGLMAction>;\n\n  try {\n    parsedResponse = parseAutoGLMResponse(rawResponse);\n    debug('thinking in response:', parsedResponse.think);\n    debug('action in response:', parsedResponse.content);\n\n    const parsedAction = parseAction(parsedResponse);\n    debug('Parsed action object:', parsedAction);\n    transformedActions = transformAutoGLMAction(\n      parsedAction,\n      context.shotSize,\n      options.actionSpace,\n    );\n    debug('Transformed actions:', transformedActions);\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `Parse error: ${errorMessage}`,\n      JSON.stringify(rawResponse, undefined, 2),\n      usage,\n    );\n  }\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: `<think>${parsedResponse.think}</think><answer>${parsedResponse.content}</answer>`,\n  });\n\n  const shouldContinuePlanning = !parsedResponse.content.startsWith('finish(');\n\n  return {\n    actions: transformedActions,\n    log: rawResponse,\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(rawResponse, undefined, 2),\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","autoGLMPlanning","userInstruction","options","conversationHistory","context","modelConfig","actionContext","systemPrompt","getAutoGLMPlanPrompt","imagePayloadBase64","memoriesText","msgs","rawResponse","usage","callAIWithStringResponse","parsedResponse","transformedActions","parseAutoGLMResponse","parsedAction","parseAction","transformAutoGLMAction","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","shouldContinuePlanning"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACQA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEhB,eAAeC,gBACpBC,eAAuB,EACvBC,OAOC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAEC,aAAa,EAAE,GAAGJ;IAErE,MAAMK,eACJC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBH,YAAY,WAAW,IAC3CC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,0BAA0B,CAAC,GACrE,EAAC;IAEP,MAAMG,qBAAqBL,QAAQ,UAAU,CAAC,MAAM;IACpD,MAAMM,eAAeP,oBAAoB,cAAc;IAEvDA,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMO,eACF,GAAGA,aAAa,EAAE,EAAET,iBAAiB,GACrCA;YACN;SACD;IACH;IACAE,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YAAC;gBAAE,MAAM;gBAAa,WAAW;oBAAE,KAAKM;gBAAmB;YAAE;SAAE;IAC1E;IAEA,MAAME,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASJ;QAAa;WACrCJ,oBAAoB,QAAQ,CAAC;KACjC;IAED,MAAM,EAAE,SAASS,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAC5CH,MACAN,aACA;QACE,aAAaH,QAAQ,WAAW;IAClC;IAGFJ,MAAM,gCAAgCc;IAEtC,IAAIG;IACJ,IAAIC;IAEJ,IAAI;QACFD,iBAAiBE,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBL;QACtCd,MAAM,yBAAyBiB,eAAe,KAAK;QACnDjB,MAAM,uBAAuBiB,eAAe,OAAO;QAEnD,MAAMG,eAAeC,AAAAA,IAAAA,mCAAAA,WAAAA,AAAAA,EAAYJ;QACjCjB,MAAM,yBAAyBoB;QAC/BF,qBAAqBI,AAAAA,IAAAA,oCAAAA,sBAAAA,AAAAA,EACnBF,cACAd,QAAQ,QAAQ,EAChBF,QAAQ,WAAW;QAErBJ,MAAM,wBAAwBkB;IAChC,EAAE,OAAOK,YAAY;QAEnB,MAAMC,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;QAC5D,MAAM,IAAII,yBAAAA,oBAAoBA,CAC5B,CAAC,aAAa,EAAEH,cAAc,EAC9BI,KAAK,SAAS,CAACd,aAAae,QAAW,IACvCd;IAEJ;IAEAV,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS,CAAC,OAAO,EAAEY,eAAe,KAAK,CAAC,gBAAgB,EAAEA,eAAe,OAAO,CAAC,SAAS,CAAC;IAC7F;IAEA,MAAMa,yBAAyB,CAACb,eAAe,OAAO,CAAC,UAAU,CAAC;IAElE,OAAO;QACL,SAASC;QACT,KAAKJ;QACLC;QACAe;QACA,aAAaF,KAAK,SAAS,CAACd,aAAae,QAAW;IACtD;AACF"}

package/dist/lib/ai-model/auto-glm/prompt.js

@@ -13,7 +13,7 @@ var __webpack_require__ = {};
 })();
 (()=>{
     __webpack_require__.r = (exports1)=>{
-        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
             value: 'Module'
         });
         Object.defineProperty(exports1, '__esModule', {