@midscene/core 1.0.1-beta-20251024064637.0 → 1.0.1-beta-20251027033034.0

package/dist/es/agent/ui-utils.mjs

@@ -1,5 +1,8 @@
 function typeStr(task) {
-    return task.subType && 'Plan' !== task.subType ? `${task.type} / ${task.subType || ''}` : task.type;
+    if ('Insight' === task.type && ('Query' === task.subType || 'Assert' === task.subType)) return task.type;
+    if ('Action' === task.type && task.subType) return `Action Space / ${task.subType}`;
+    if (task.subType) return `${task.type} / ${task.subType}`;
+    return task.type;
 }
 function locateParamStr(locate) {
     if (!locate) return '';

package/dist/es/agent/ui-utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-utils.mjs","sources":["webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightLocate,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  return task.subType && task.subType !== 'Plan'\n    ? `${task.type} / ${task.subType || ''}`\n    : task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n      const prompt = locate.prompt.prompt;\n      const images = locate.prompt.images || [];\n\n      if (images.length === 0) return prompt;\n\n      const imagesStr = images\n        .map((image) => {\n          let url = image.url;\n          if (\n            url.startsWith('data:image/') ||\n            (url.startsWith('data:') && url.includes('base64'))\n          ) {\n            url = `${url.substring(0, 15)}...`;\n          }\n          return `[${image.name}](${url})`;\n        })\n        .join(', ');\n\n      return `${prompt}, ${imagesStr}`;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      locateParamStr((task as ExecutionTaskInsightLocate)?.param) ||\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action') {\n    const locate = (task as ExecutionTaskAction)?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object' && locateParamStr(value as any)) {\n    return locateParamStr(value as any);\n  }\n\n  return JSON.stringify(value, undefined, 2);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","images","imagesStr","image","url","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","paramStr","value","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","locateStr","_task_param7","_task_param8","JSON","undefined"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IACzC,OAAOA,KAAK,OAAO,IAAIA,AAAiB,WAAjBA,KAAK,OAAO,GAC/B,GAAGA,KAAK,IAAI,CAAC,GAAG,EAAEA,KAAK,OAAO,IAAI,IAAI,GACtCA,KAAK,IAAI;AACf;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAAE;YAC7D,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,MAAME,SAASF,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE;YAEzC,IAAIE,AAAkB,MAAlBA,OAAO,MAAM,EAAQ,OAAOD;YAEhC,MAAME,YAAYD,OACf,GAAG,CAAC,CAACE;gBACJ,IAAIC,MAAMD,MAAM,GAAG;gBACnB,IACEC,IAAI,UAAU,CAAC,kBACdA,IAAI,UAAU,CAAC,YAAYA,IAAI,QAAQ,CAAC,WAEzCA,MAAM,GAAGA,IAAI,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC;gBAEpC,OAAO,CAAC,CAAC,EAAED,MAAM,IAAI,CAAC,EAAE,EAAEC,IAAI,CAAC,CAAC;YAClC,GACC,IAAI,CAAC;YAER,OAAO,GAAGJ,OAAO,EAAE,EAAEE,WAAW;QAClC;IACF;IAEA,OAAO;AACT;AAEO,SAASG,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,OAAO,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACjI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZX,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGW,KAAK,GAAG,EAAEX,QAAQ;IAE9B,OAAOW;AACT;AAEO,SAASC,SAASf,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACpBiB;QAARD,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAADiB,CAAAA,cAACjB,KAAgC,KAAK,AAAD,IAArCiB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAEA,IAAIjB,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAGzBkB,cACAC;QAHFH,QACEf,eAAgBD,QAAAA,OAAAA,KAAAA,IAAAA,KAAqC,KAAK,KACzDA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADkB,CAAAA,eAAClB,KAAoC,KAAK,AAAD,IAAzCkB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDlB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,eAACnB,KAAwC,KAAK,AAAD,IAA7CmB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAInB,AAAc,aAAdA,KAAK,IAAI,EAAe;YAKfoB,cAGFC,cAIAC,cAKAC;QAhBT,MAAMrB,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,MAAM;QACpD,MAAMwB,YAAYtB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAA8B,KAAK,AAAD,IAAnCoB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXT,QAAQ,GAAIhB,QAAAA,OAAAA,KAAAA,IAAAA,QAADyB,CAAAA,eAACzB,KAA8B,KAAK,AAAD,IAAnCyB,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQzB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAA8B,KAAK,AAAD,IAAnCqB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDL,QAAQR,eAAgBR,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADsB,CAAAA,eAACtB,KAA8B,KAAK,AAAD,IAAnCsB,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAACtB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CgB,QAAQN,aAAcV,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAA8B,KAAK,AAAD,IAAnCuB,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARV,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,KAAK;QACrD;QAEA,IAAIF,WAEAR,QADEA,QACM,GAAGQ,UAAU,GAAG,EAAER,OAAO,GAEzBQ;IAGd;IAEA,IAAI,AAAiB,WAAVR,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,SAAsBf,eAAee,QAC9C,OAAOf,eAAee;IAGxB,OAAOW,KAAK,SAAS,CAACX,OAAOY,QAAW;AAC1C"}
+{"version":3,"file":"agent/ui-utils.mjs","sources":["webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightLocate,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // For Insight tasks with Query or Assert subtypes, show just \"Insight\"\n  if (\n    task.type === 'Insight' &&\n    (task.subType === 'Query' || task.subType === 'Assert')\n  ) {\n    return task.type;\n  }\n\n  // For Action tasks with subType, show \"Action Space / subType\"\n  if (task.type === 'Action' && task.subType) {\n    return `Action Space / ${task.subType}`;\n  }\n\n  // For all other cases with subType, show \"type / subType\"\n  if (task.subType) {\n    return `${task.type} / ${task.subType}`;\n  }\n\n  // No subType, just show type\n  return task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n      const prompt = locate.prompt.prompt;\n      const images = locate.prompt.images || [];\n\n      if (images.length === 0) return prompt;\n\n      const imagesStr = images\n        .map((image) => {\n          let url = image.url;\n          if (\n            url.startsWith('data:image/') ||\n            (url.startsWith('data:') && url.includes('base64'))\n          ) {\n            url = `${url.substring(0, 15)}...`;\n          }\n          return `[${image.name}](${url})`;\n        })\n        .join(', ');\n\n      return `${prompt}, ${imagesStr}`;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      locateParamStr((task as ExecutionTaskInsightLocate)?.param) ||\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action') {\n    const locate = (task as ExecutionTaskAction)?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object' && locateParamStr(value as any)) {\n    return locateParamStr(value as any);\n  }\n\n  return JSON.stringify(value, undefined, 2);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","images","imagesStr","image","url","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","paramStr","value","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","locateStr","_task_param7","_task_param8","JSON","undefined"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IAEzC,IACEA,AAAc,cAAdA,KAAK,IAAI,IACRA,CAAAA,AAAiB,YAAjBA,KAAK,OAAO,IAAgBA,AAAiB,aAAjBA,KAAK,OAAO,AAAY,GAErD,OAAOA,KAAK,IAAI;IAIlB,IAAIA,AAAc,aAAdA,KAAK,IAAI,IAAiBA,KAAK,OAAO,EACxC,OAAO,CAAC,eAAe,EAAEA,KAAK,OAAO,EAAE;IAIzC,IAAIA,KAAK,OAAO,EACd,OAAO,GAAGA,KAAK,IAAI,CAAC,GAAG,EAAEA,KAAK,OAAO,EAAE;IAIzC,OAAOA,KAAK,IAAI;AAClB;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAAE;YAC7D,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,MAAME,SAASF,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE;YAEzC,IAAIE,AAAkB,MAAlBA,OAAO,MAAM,EAAQ,OAAOD;YAEhC,MAAME,YAAYD,OACf,GAAG,CAAC,CAACE;gBACJ,IAAIC,MAAMD,MAAM,GAAG;gBACnB,IACEC,IAAI,UAAU,CAAC,kBACdA,IAAI,UAAU,CAAC,YAAYA,IAAI,QAAQ,CAAC,WAEzCA,MAAM,GAAGA,IAAI,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC;gBAEpC,OAAO,CAAC,CAAC,EAAED,MAAM,IAAI,CAAC,EAAE,EAAEC,IAAI,CAAC,CAAC;YAClC,GACC,IAAI,CAAC;YAER,OAAO,GAAGJ,OAAO,EAAE,EAAEE,WAAW;QAClC;IACF;IAEA,OAAO;AACT;AAEO,SAASG,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,OAAO,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACjI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZX,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGW,KAAK,GAAG,EAAEX,QAAQ;IAE9B,OAAOW;AACT;AAEO,SAASC,SAASf,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACpBiB;QAARD,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAADiB,CAAAA,cAACjB,KAAgC,KAAK,AAAD,IAArCiB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAEA,IAAIjB,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAGzBkB,cACAC;QAHFH,QACEf,eAAgBD,QAAAA,OAAAA,KAAAA,IAAAA,KAAqC,KAAK,KACzDA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADkB,CAAAA,eAAClB,KAAoC,KAAK,AAAD,IAAzCkB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDlB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,eAACnB,KAAwC,KAAK,AAAD,IAA7CmB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAInB,AAAc,aAAdA,KAAK,IAAI,EAAe;YAKfoB,cAGFC,cAIAC,cAKAC;QAhBT,MAAMrB,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,MAAM;QACpD,MAAMwB,YAAYtB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAA8B,KAAK,AAAD,IAAnCoB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXT,QAAQ,GAAIhB,QAAAA,OAAAA,KAAAA,IAAAA,QAADyB,CAAAA,eAACzB,KAA8B,KAAK,AAAD,IAAnCyB,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQzB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAA8B,KAAK,AAAD,IAAnCqB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDL,QAAQR,eAAgBR,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADsB,CAAAA,eAACtB,KAA8B,KAAK,AAAD,IAAnCsB,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAACtB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CgB,QAAQN,aAAcV,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAA8B,KAAK,AAAD,IAAnCuB,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARV,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,KAAK;QACrD;QAEA,IAAIF,WAEAR,QADEA,QACM,GAAGQ,UAAU,GAAG,EAAER,OAAO,GAEzBQ;IAGd;IAEA,IAAI,AAAiB,WAAVR,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,SAAsBf,eAAee,QAC9C,OAAOf,eAAee;IAGxB,OAAOW,KAAK,SAAS,CAACX,OAAOY,QAAW;AAC1C"}

package/dist/es/agent/utils.mjs

@@ -99,7 +99,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.0.1-beta-20251024064637.0";
+const getMidsceneVersion = ()=>"1.0.1-beta-20251027033034.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/service-caller/index.mjs

@@ -11,37 +11,29 @@ import { assertSchema } from "../prompt/assertion.mjs";
 import { planSchema } from "../prompt/llm-planning.mjs";
 async function createChatClient({ AIActionTypeValue, modelConfig }) {
     const { socksProxy, httpProxy, modelName, openaiBaseURL, openaiApiKey, openaiExtraConfig, modelDescription, uiTarsModelVersion: uiTarsVersion, vlMode, createOpenAIClient } = modelConfig;
-    let openai;
-    if (createOpenAIClient) openai = createOpenAIClient({
-        modelName,
-        openaiApiKey,
-        openaiBaseURL,
-        socksProxy,
-        httpProxy,
-        openaiExtraConfig,
-        vlMode,
-        intent: modelConfig.intent,
-        modelDescription
-    });
-    else {
-        let proxyAgent;
-        const debugProxy = getDebug('ai:call:proxy');
-        if (httpProxy) {
-            debugProxy('using http proxy', httpProxy);
-            proxyAgent = new HttpsProxyAgent(httpProxy);
-        } else if (socksProxy) {
-            debugProxy('using socks proxy', socksProxy);
-            proxyAgent = new SocksProxyAgent(socksProxy);
-        }
-        openai = new openai_0({
-            baseURL: openaiBaseURL,
-            apiKey: openaiApiKey,
-            ...proxyAgent ? {
-                httpAgent: proxyAgent
-            } : {},
-            ...openaiExtraConfig,
-            dangerouslyAllowBrowser: true
-        });
+    let proxyAgent;
+    const debugProxy = getDebug('ai:call:proxy');
+    if (httpProxy) {
+        debugProxy('using http proxy', httpProxy);
+        proxyAgent = new HttpsProxyAgent(httpProxy);
+    } else if (socksProxy) {
+        debugProxy('using socks proxy', socksProxy);
+        proxyAgent = new SocksProxyAgent(socksProxy);
+    }
+    const openAIOptions = {
+        baseURL: openaiBaseURL,
+        apiKey: openaiApiKey,
+        ...proxyAgent ? {
+            httpAgent: proxyAgent
+        } : {},
+        ...openaiExtraConfig,
+        dangerouslyAllowBrowser: true
+    };
+    const baseOpenAI = new openai_0(openAIOptions);
+    let openai = baseOpenAI;
+    if (createOpenAIClient) {
+        const wrappedClient = await createOpenAIClient(baseOpenAI, openAIOptions);
+        if (wrappedClient) openai = wrappedClient;
     }
     return {
         completion: openai.chat.completions,

package/dist/es/ai-model/service-caller/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport {\n  type IModelConfig,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelConfig,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n    createOpenAIClient,\n  } = modelConfig;\n\n  let openai: OpenAI;\n\n  // Use custom client factory if provided\n  if (createOpenAIClient) {\n    openai = createOpenAIClient({\n      modelName,\n      openaiApiKey,\n      openaiBaseURL,\n      socksProxy,\n      httpProxy,\n      openaiExtraConfig,\n      vlMode,\n      intent: modelConfig.intent,\n      modelDescription,\n    });\n  } else {\n    // Default OpenAI client creation\n    let proxyAgent = undefined;\n    const debugProxy = getDebug('ai:call:proxy');\n    if (httpProxy) {\n      debugProxy('using http proxy', httpProxy);\n      proxyAgent = new HttpsProxyAgent(httpProxy);\n    } else if (socksProxy) {\n      debugProxy('using socks proxy', socksProxy);\n      proxyAgent = new SocksProxyAgent(socksProxy);\n    }\n\n    openai = new OpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      ...(proxyAgent ? { httpAgent: proxyAgent as any } : {}),\n      ...openaiExtraConfig,\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  return {\n    completion: openai.chat.completions,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  };\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const { completion, modelName, modelDescription, uiTarsVersion, vlMode } =\n    await createChatClient({\n      AIActionTypeValue,\n      modelConfig,\n    });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens = globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlMode === 'vlm-ui-tars' ? 0.0 : 0.1,\n    stream: !!isStreaming,\n    max_tokens:\n      typeof maxTokens === 'number'\n        ? maxTokens\n        : Number.parseInt(maxTokens || '2048', 10),\n    ...(vlMode === 'qwen-vl' // qwen vl v2 specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    debugCall(\n      `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n    );\n\n    if (isStreaming) {\n      const stream = (await completion.create(\n        {\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        },\n        {\n          stream: true,\n        },\n      )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n        _request_id?: string | null;\n      };\n\n      for await (const chunk of stream) {\n        const content = chunk.choices?.[0]?.delta?.content || '';\n        const reasoning_content =\n          (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n        // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n        if (chunk.usage) {\n          usage = chunk.usage;\n        }\n\n        if (content || reasoning_content) {\n          accumulated += content;\n          const chunkData: CodeGenerationChunk = {\n            content,\n            reasoning_content,\n            accumulated,\n            isComplete: false,\n            usage: undefined,\n          };\n          options.onChunk!(chunkData);\n        }\n\n        // Check if stream is complete\n        if (chunk.choices?.[0]?.finish_reason) {\n          timeCost = Date.now() - startTime;\n\n          // If usage is not available from the stream, provide a basic usage info\n          if (!usage) {\n            // Estimate token counts based on content length (rough approximation)\n            const estimatedTokens = Math.max(\n              1,\n              Math.floor(accumulated.length / 4),\n            );\n            usage = {\n              prompt_tokens: estimatedTokens,\n              completion_tokens: estimatedTokens,\n              total_tokens: estimatedTokens * 2,\n            };\n          }\n\n          // Send final chunk\n          const finalChunk: CodeGenerationChunk = {\n            content: '',\n            accumulated,\n            reasoning_content: '',\n            isComplete: true,\n            usage: {\n              prompt_tokens: usage.prompt_tokens ?? 0,\n              completion_tokens: usage.completion_tokens ?? 0,\n              total_tokens: usage.total_tokens ?? 0,\n              time_cost: timeCost ?? 0,\n              model_name: modelName,\n              model_description: modelDescription,\n              intent: modelConfig.intent,\n            },\n          };\n          options.onChunk!(finalChunk);\n          break;\n        }\n      }\n      content = accumulated;\n      debugProfileStats(\n        `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`,\n      );\n    } else {\n      const result = await completion.create({\n        model: modelName,\n        messages,\n        response_format: responseFormat,\n        ...commonConfig,\n      } as any);\n      timeCost = Date.now() - startTime;\n\n      debugProfileStats(\n        `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n      );\n\n      debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);\n\n      assert(\n        result.choices,\n        `invalid response from LLM service: ${JSON.stringify(result)}`,\n      );\n      content = result.choices[0].message.content!;\n      usage = result.usage;\n    }\n\n    debugCall(`response: ${content}`);\n    assert(content, 'empty content');\n\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: modelName,\n            model_description: modelDescription,\n            intent: modelConfig.intent,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n      case AIActionType.TEXT:\n        // No response format for plain text - return as-is\n        responseFormat = undefined;\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  // Skip for plain text to allow string output\n  if (\n    modelName === 'gpt-4o-2024-05-13' &&\n    AIActionTypeValue !== AIActionType.TEXT\n  ) {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callAIWithObjectResponse<T>(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const response = await callAI(messages, AIActionTypeValue, modelConfig);\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  return { content: jsonContent, usage: response.usage };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n  try {\n    return JSON.parse(cleanJsonString);\n  } catch {}\n  try {\n    return JSON.parse(jsonrepair(cleanJsonString));\n  } catch (e) {}\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    return JSON.parse(jsonrepair(jsonString));\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["createChatClient","AIActionTypeValue","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","modelDescription","uiTarsVersion","vlMode","createOpenAIClient","openai","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","OpenAI","callAI","messages","options","completion","responseFormat","getResponseFormat","maxTokens","globalConfigManager","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","JSON","assert","e","console","newError","Error","AIActionType","assertSchema","planSchema","AIResponseFormat","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;AAsBA,eAAeA,iBAAiB,EAC9BC,iBAAiB,EACjBC,WAAW,EAIZ;IAOC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACNC,kBAAkB,EACnB,GAAGV;IAEJ,IAAIW;IAGJ,IAAID,oBACFC,SAASD,mBAAmB;QAC1BP;QACAE;QACAD;QACAH;QACAC;QACAI;QACAG;QACA,QAAQT,YAAY,MAAM;QAC1BO;IACF;SACK;QAEL,IAAIK;QACJ,MAAMC,aAAaC,SAAS;QAC5B,IAAIZ,WAAW;YACbW,WAAW,oBAAoBX;YAC/BU,aAAa,IAAIG,gBAAgBb;QACnC,OAAO,IAAID,YAAY;YACrBY,WAAW,qBAAqBZ;YAChCW,aAAa,IAAII,gBAAgBf;QACnC;QAEAU,SAAS,IAAIM,SAAO;YAClB,SAASb;YACT,QAAQC;YACR,GAAIO,aAAa;gBAAE,WAAWA;YAAkB,IAAI,CAAC,CAAC;YACtD,GAAGN,iBAAiB;YACpB,yBAAyB;QAC3B;IACF;IAEA,OAAO;QACL,YAAYK,OAAO,IAAI,CAAC,WAAW;QACnCR;QACAI;QACAC;QACAC;IACF;AACF;AAEO,eAAeS,OACpBC,QAAsC,EACtCpB,iBAA+B,EAC/BC,WAAyB,EACzBoB,OAGC;IAED,MAAM,EAAEC,UAAU,EAAElB,SAAS,EAAEI,gBAAgB,EAAEC,aAAa,EAAEC,MAAM,EAAE,GACtE,MAAMX,iBAAiB;QACrBC;QACAC;IACF;IAEF,MAAMsB,iBAAiBC,kBAAkBpB,WAAWJ;IAEpD,MAAMyB,YAAYC,oBAAoB,iBAAiB,CAACC;IACxD,MAAMC,YAAYb,SAAS;IAC3B,MAAMc,oBAAoBd,SAAS;IACnC,MAAMe,qBAAqBf,SAAS;IAEpC,MAAMgB,YAAYC,KAAK,GAAG;IAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIa;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,eAAe;QACnB,aAAa5B,AAAW,kBAAXA,SAA2B,MAAM;QAC9C,QAAQ,CAAC,CAACuB;QACV,YACE,AAAqB,YAArB,OAAOR,YACHA,YACAc,OAAO,QAAQ,CAACd,aAAa,QAAQ;QAC3C,GAAIf,AAAW,cAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACFkB,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAE7B,WAAW;QAGrE,IAAI6B,aAAa;YACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CACrC;gBACE,OAAOlB;gBACPgB;gBACA,iBAAiBG;gBACjB,GAAGe,YAAY;YACjB,GACA;gBACE,QAAQ;YACV;YAKF,WAAW,MAAMG,SAASD,OAAQ;oBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;gBAtBJ,MAAMV,UAAUQ,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;gBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;gBAG3D,IAAIF,MAAM,KAAK,EACbL,QAAQK,MAAM,KAAK;gBAGrB,IAAIP,WAAWW,mBAAmB;oBAChCV,eAAeD;oBACf,MAAMY,YAAiC;wBACrCZ;wBACAW;wBACAV;wBACA,YAAY;wBACZ,OAAOY;oBACT;oBACA1B,QAAQ,OAAO,CAAEyB;gBACnB;gBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;oBACrCP,WAAWL,KAAK,GAAG,KAAKD;oBAGxB,IAAI,CAACK,OAAO;wBAEV,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACd,YAAY,MAAM,GAAG;wBAElCC,QAAQ;4BACN,eAAeY;4BACf,mBAAmBA;4BACnB,cAAcA,AAAkB,IAAlBA;wBAChB;oBACF;oBAGA,MAAME,aAAkC;wBACtC,SAAS;wBACTf;wBACA,mBAAmB;wBACnB,YAAY;wBACZ,OAAO;4BACL,eAAeC,MAAM,aAAa,IAAI;4BACtC,mBAAmBA,MAAM,iBAAiB,IAAI;4BAC9C,cAAcA,MAAM,YAAY,IAAI;4BACpC,WAAWC,YAAY;4BACvB,YAAYjC;4BACZ,mBAAmBI;4BACnB,QAAQP,YAAY,MAAM;wBAC5B;oBACF;oBACAoB,QAAQ,OAAO,CAAE6B;oBACjB;gBACF;YACF;YACAhB,UAAUC;YACVN,kBACE,CAAC,iBAAiB,EAAEzB,UAAU,QAAQ,EAAEM,UAAU,UAAU,WAAW,EAAE2B,UAAU;QAEvF,OAAO;gBAUqGc,eAAyDC,gBAAwDC;YAT3N,MAAMC,SAAS,MAAMhC,WAAW,MAAM,CAAC;gBACrC,OAAOlB;gBACPgB;gBACA,iBAAiBG;gBACjB,GAAGe,YAAY;YACjB;YACAD,WAAWL,KAAK,GAAG,KAAKD;YAExBF,kBACE,CAAC,OAAO,EAAEzB,UAAU,QAAQ,EAAEM,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAE0C,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEhB,SAAS,aAAa,EAAEiB,OAAO,WAAW,IAAI,IAAI;YAG3TxB,mBAAmB,CAAC,oBAAoB,EAAEyB,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;YAExEE,OACEF,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;YAEhEpB,UAAUoB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;YAC3ClB,QAAQkB,OAAO,KAAK;QACtB;QAEA1B,UAAU,CAAC,UAAU,EAAEM,SAAS;QAChCsB,OAAOtB,SAAS;QAGhB,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEf,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAeY;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASd,WAAW;YACpB,OAAOE,QACH;gBACE,eAAeA,MAAM,aAAa,IAAI;gBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gBAC9C,cAAcA,MAAM,YAAY,IAAI;gBACpC,WAAWC,YAAY;gBACvB,YAAYjC;gBACZ,mBAAmBI;gBACnB,QAAQP,YAAY,MAAM;YAC5B,IACA8C;YACJ,YAAY,CAAC,CAACd;QAChB;IACF,EAAE,OAAOwB,GAAQ;QACfC,QAAQ,KAAK,CAAC,kBAAkBD;QAChC,MAAME,WAAW,IAAIC,MACnB,CAAC,eAAe,EAAE3B,cAAc,eAAe,GAAG,kBAAkB,EAAEwB,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAME;IACR;AACF;AAEO,MAAMnC,oBAAoB,CAC/BpB,WACAJ;IAIA,IAAIuB;IAKJ,IAAInB,UAAU,QAAQ,CAAC,UACrB,OAAQJ;QACN,KAAK6D,aAAa,MAAM;YACtBtC,iBAAiBuC;YACjB;QACF,KAAKD,aAAa,IAAI;YACpBtC,iBAAiBwC;YACjB;QACF,KAAKF,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChCtC,iBAAiB;gBAAE,MAAMyC,iBAAiB,IAAI;YAAC;YAC/C;QACF,KAAKH,aAAa,IAAI;YAEpBtC,iBAAiBwB;YACjB;IACJ;IAKF,IACE3C,AAAc,wBAAdA,aACAJ,sBAAsB6D,aAAa,IAAI,EAEvCtC,iBAAiB;QAAE,MAAMyC,iBAAiB,IAAI;IAAC;IAGjD,OAAOzC;AACT;AAEO,eAAe0C,yBACpB7C,QAAsC,EACtCpB,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAMiE,WAAW,MAAM/C,OAAOC,UAAUpB,mBAAmBC;IAC3DuD,OAAOU,UAAU;IACjB,MAAMxD,SAAST,YAAY,MAAM;IACjC,MAAMkE,cAAcC,cAAcF,SAAS,OAAO,EAAExD;IACpD,OAAO;QAAE,SAASyD;QAAa,OAAOD,SAAS,KAAK;IAAC;AACvD;AAEO,eAAeG,yBACpBC,IAAY,EACZtE,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM,EAAEiC,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,OAAOmD,MAAMtE,mBAAmBC;IACjE,OAAO;QAAEiC;QAASE;IAAM;AAC1B;AAEO,SAASmC,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASR,cAAcQ,KAAa,EAAElE,MAAgC;IAC3E,MAAMmE,kBAAkBN,yBAAyBK;IAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAACvC;IACT;IACA,IAAI;QACF,OAAOgB,KAAK,KAAK,CAACsB;IACpB,EAAE,OAAM,CAAC;IACT,IAAI;QACF,OAAOtB,KAAK,KAAK,CAACwB,WAAWF;IAC/B,EAAE,OAAOpB,GAAG,CAAC;IAEb,IAAI/C,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAMsE,aAAaL,yBAAyBE;QAC5C,OAAOtB,KAAK,KAAK,CAACwB,WAAWC;IAC/B;IACA,MAAMpB,MAAM,CAAC,+BAA+B,EAAEgB,OAAO;AACvD"}
+{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport {\n  type IModelConfig,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelConfig,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n    createOpenAIClient,\n  } = modelConfig;\n\n  let proxyAgent = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n  if (httpProxy) {\n    debugProxy('using http proxy', httpProxy);\n    proxyAgent = new HttpsProxyAgent(httpProxy);\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', socksProxy);\n    proxyAgent = new SocksProxyAgent(socksProxy);\n  }\n\n  const openAIOptions = {\n    baseURL: openaiBaseURL,\n    apiKey: openaiApiKey,\n    ...(proxyAgent ? { httpAgent: proxyAgent as any } : {}),\n    ...openaiExtraConfig,\n    dangerouslyAllowBrowser: true,\n  };\n\n  const baseOpenAI = new OpenAI(openAIOptions);\n\n  let openai: OpenAI = baseOpenAI;\n\n  if (createOpenAIClient) {\n    const wrappedClient = await createOpenAIClient(baseOpenAI, openAIOptions);\n\n    if (wrappedClient) {\n      openai = wrappedClient as OpenAI;\n    }\n  }\n\n  return {\n    completion: openai.chat.completions,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  };\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const { completion, modelName, modelDescription, uiTarsVersion, vlMode } =\n    await createChatClient({\n      AIActionTypeValue,\n      modelConfig,\n    });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens = globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlMode === 'vlm-ui-tars' ? 0.0 : 0.1,\n    stream: !!isStreaming,\n    max_tokens:\n      typeof maxTokens === 'number'\n        ? maxTokens\n        : Number.parseInt(maxTokens || '2048', 10),\n    ...(vlMode === 'qwen-vl' // qwen vl v2 specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    debugCall(\n      `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n    );\n\n    if (isStreaming) {\n      const stream = (await completion.create(\n        {\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        },\n        {\n          stream: true,\n        },\n      )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n        _request_id?: string | null;\n      };\n\n      for await (const chunk of stream) {\n        const content = chunk.choices?.[0]?.delta?.content || '';\n        const reasoning_content =\n          (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n        // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n        if (chunk.usage) {\n          usage = chunk.usage;\n        }\n\n        if (content || reasoning_content) {\n          accumulated += content;\n          const chunkData: CodeGenerationChunk = {\n            content,\n            reasoning_content,\n            accumulated,\n            isComplete: false,\n            usage: undefined,\n          };\n          options.onChunk!(chunkData);\n        }\n\n        // Check if stream is complete\n        if (chunk.choices?.[0]?.finish_reason) {\n          timeCost = Date.now() - startTime;\n\n          // If usage is not available from the stream, provide a basic usage info\n          if (!usage) {\n            // Estimate token counts based on content length (rough approximation)\n            const estimatedTokens = Math.max(\n              1,\n              Math.floor(accumulated.length / 4),\n            );\n            usage = {\n              prompt_tokens: estimatedTokens,\n              completion_tokens: estimatedTokens,\n              total_tokens: estimatedTokens * 2,\n            };\n          }\n\n          // Send final chunk\n          const finalChunk: CodeGenerationChunk = {\n            content: '',\n            accumulated,\n            reasoning_content: '',\n            isComplete: true,\n            usage: {\n              prompt_tokens: usage.prompt_tokens ?? 0,\n              completion_tokens: usage.completion_tokens ?? 0,\n              total_tokens: usage.total_tokens ?? 0,\n              time_cost: timeCost ?? 0,\n              model_name: modelName,\n              model_description: modelDescription,\n              intent: modelConfig.intent,\n            },\n          };\n          options.onChunk!(finalChunk);\n          break;\n        }\n      }\n      content = accumulated;\n      debugProfileStats(\n        `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`,\n      );\n    } else {\n      const result = await completion.create({\n        model: modelName,\n        messages,\n        response_format: responseFormat,\n        ...commonConfig,\n      } as any);\n      timeCost = Date.now() - startTime;\n\n      debugProfileStats(\n        `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n      );\n\n      debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);\n\n      assert(\n        result.choices,\n        `invalid response from LLM service: ${JSON.stringify(result)}`,\n      );\n      content = result.choices[0].message.content!;\n      usage = result.usage;\n    }\n\n    debugCall(`response: ${content}`);\n    assert(content, 'empty content');\n\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: modelName,\n            model_description: modelDescription,\n            intent: modelConfig.intent,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n      case AIActionType.TEXT:\n        // No response format for plain text - return as-is\n        responseFormat = undefined;\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  // Skip for plain text to allow string output\n  if (\n    modelName === 'gpt-4o-2024-05-13' &&\n    AIActionTypeValue !== AIActionType.TEXT\n  ) {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callAIWithObjectResponse<T>(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const response = await callAI(messages, AIActionTypeValue, modelConfig);\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  return { content: jsonContent, usage: response.usage };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n  try {\n    return JSON.parse(cleanJsonString);\n  } catch {}\n  try {\n    return JSON.parse(jsonrepair(cleanJsonString));\n  } catch (e) {}\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    return JSON.parse(jsonrepair(jsonString));\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["createChatClient","AIActionTypeValue","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","modelDescription","uiTarsVersion","vlMode","createOpenAIClient","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","openAIOptions","baseOpenAI","OpenAI","openai","wrappedClient","callAI","messages","options","completion","responseFormat","getResponseFormat","maxTokens","globalConfigManager","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","JSON","assert","e","console","newError","Error","AIActionType","assertSchema","planSchema","AIResponseFormat","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;AAsBA,eAAeA,iBAAiB,EAC9BC,iBAAiB,EACjBC,WAAW,EAIZ;IAOC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACNC,kBAAkB,EACnB,GAAGV;IAEJ,IAAIW;IACJ,MAAMC,aAAaC,SAAS;IAC5B,IAAIX,WAAW;QACbU,WAAW,oBAAoBV;QAC/BS,aAAa,IAAIG,gBAAgBZ;IACnC,OAAO,IAAID,YAAY;QACrBW,WAAW,qBAAqBX;QAChCU,aAAa,IAAII,gBAAgBd;IACnC;IAEA,MAAMe,gBAAgB;QACpB,SAASZ;QACT,QAAQC;QACR,GAAIM,aAAa;YAAE,WAAWA;QAAkB,IAAI,CAAC,CAAC;QACtD,GAAGL,iBAAiB;QACpB,yBAAyB;IAC3B;IAEA,MAAMW,aAAa,IAAIC,SAAOF;IAE9B,IAAIG,SAAiBF;IAErB,IAAIP,oBAAoB;QACtB,MAAMU,gBAAgB,MAAMV,mBAAmBO,YAAYD;QAE3D,IAAII,eACFD,SAASC;IAEb;IAEA,OAAO;QACL,YAAYD,OAAO,IAAI,CAAC,WAAW;QACnChB;QACAI;QACAC;QACAC;IACF;AACF;AAEO,eAAeY,OACpBC,QAAsC,EACtCvB,iBAA+B,EAC/BC,WAAyB,EACzBuB,OAGC;IAED,MAAM,EAAEC,UAAU,EAAErB,SAAS,EAAEI,gBAAgB,EAAEC,aAAa,EAAEC,MAAM,EAAE,GACtE,MAAMX,iBAAiB;QACrBC;QACAC;IACF;IAEF,MAAMyB,iBAAiBC,kBAAkBvB,WAAWJ;IAEpD,MAAM4B,YAAYC,oBAAoB,iBAAiB,CAACC;IACxD,MAAMC,YAAYjB,SAAS;IAC3B,MAAMkB,oBAAoBlB,SAAS;IACnC,MAAMmB,qBAAqBnB,SAAS;IAEpC,MAAMoB,YAAYC,KAAK,GAAG;IAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIa;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,eAAe;QACnB,aAAa/B,AAAW,kBAAXA,SAA2B,MAAM;QAC9C,QAAQ,CAAC,CAAC0B;QACV,YACE,AAAqB,YAArB,OAAOR,YACHA,YACAc,OAAO,QAAQ,CAACd,aAAa,QAAQ;QAC3C,GAAIlB,AAAW,cAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACFqB,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAEhC,WAAW;QAGrE,IAAIgC,aAAa;YACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CACrC;gBACE,OAAOrB;gBACPmB;gBACA,iBAAiBG;gBACjB,GAAGe,YAAY;YACjB,GACA;gBACE,QAAQ;YACV;YAKF,WAAW,MAAMG,SAASD,OAAQ;oBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;gBAtBJ,MAAMV,UAAUQ,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;gBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;gBAG3D,IAAIF,MAAM,KAAK,EACbL,QAAQK,MAAM,KAAK;gBAGrB,IAAIP,WAAWW,mBAAmB;oBAChCV,eAAeD;oBACf,MAAMY,YAAiC;wBACrCZ;wBACAW;wBACAV;wBACA,YAAY;wBACZ,OAAOY;oBACT;oBACA1B,QAAQ,OAAO,CAAEyB;gBACnB;gBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;oBACrCP,WAAWL,KAAK,GAAG,KAAKD;oBAGxB,IAAI,CAACK,OAAO;wBAEV,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACd,YAAY,MAAM,GAAG;wBAElCC,QAAQ;4BACN,eAAeY;4BACf,mBAAmBA;4BACnB,cAAcA,AAAkB,IAAlBA;wBAChB;oBACF;oBAGA,MAAME,aAAkC;wBACtC,SAAS;wBACTf;wBACA,mBAAmB;wBACnB,YAAY;wBACZ,OAAO;4BACL,eAAeC,MAAM,aAAa,IAAI;4BACtC,mBAAmBA,MAAM,iBAAiB,IAAI;4BAC9C,cAAcA,MAAM,YAAY,IAAI;4BACpC,WAAWC,YAAY;4BACvB,YAAYpC;4BACZ,mBAAmBI;4BACnB,QAAQP,YAAY,MAAM;wBAC5B;oBACF;oBACAuB,QAAQ,OAAO,CAAE6B;oBACjB;gBACF;YACF;YACAhB,UAAUC;YACVN,kBACE,CAAC,iBAAiB,EAAE5B,UAAU,QAAQ,EAAEM,UAAU,UAAU,WAAW,EAAE8B,UAAU;QAEvF,OAAO;gBAUqGc,eAAyDC,gBAAwDC;YAT3N,MAAMC,SAAS,MAAMhC,WAAW,MAAM,CAAC;gBACrC,OAAOrB;gBACPmB;gBACA,iBAAiBG;gBACjB,GAAGe,YAAY;YACjB;YACAD,WAAWL,KAAK,GAAG,KAAKD;YAExBF,kBACE,CAAC,OAAO,EAAE5B,UAAU,QAAQ,EAAEM,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAE6C,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEhB,SAAS,aAAa,EAAEiB,OAAO,WAAW,IAAI,IAAI;YAG3TxB,mBAAmB,CAAC,oBAAoB,EAAEyB,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;YAExEE,OACEF,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;YAEhEpB,UAAUoB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;YAC3ClB,QAAQkB,OAAO,KAAK;QACtB;QAEA1B,UAAU,CAAC,UAAU,EAAEM,SAAS;QAChCsB,OAAOtB,SAAS;QAGhB,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEf,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAeY;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASd,WAAW;YACpB,OAAOE,QACH;gBACE,eAAeA,MAAM,aAAa,IAAI;gBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gBAC9C,cAAcA,MAAM,YAAY,IAAI;gBACpC,WAAWC,YAAY;gBACvB,YAAYpC;gBACZ,mBAAmBI;gBACnB,QAAQP,YAAY,MAAM;YAC5B,IACAiD;YACJ,YAAY,CAAC,CAACd;QAChB;IACF,EAAE,OAAOwB,GAAQ;QACfC,QAAQ,KAAK,CAAC,kBAAkBD;QAChC,MAAME,WAAW,IAAIC,MACnB,CAAC,eAAe,EAAE3B,cAAc,eAAe,GAAG,kBAAkB,EAAEwB,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAME;IACR;AACF;AAEO,MAAMnC,oBAAoB,CAC/BvB,WACAJ;IAIA,IAAI0B;IAKJ,IAAItB,UAAU,QAAQ,CAAC,UACrB,OAAQJ;QACN,KAAKgE,aAAa,MAAM;YACtBtC,iBAAiBuC;YACjB;QACF,KAAKD,aAAa,IAAI;YACpBtC,iBAAiBwC;YACjB;QACF,KAAKF,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChCtC,iBAAiB;gBAAE,MAAMyC,iBAAiB,IAAI;YAAC;YAC/C;QACF,KAAKH,aAAa,IAAI;YAEpBtC,iBAAiBwB;YACjB;IACJ;IAKF,IACE9C,AAAc,wBAAdA,aACAJ,sBAAsBgE,aAAa,IAAI,EAEvCtC,iBAAiB;QAAE,MAAMyC,iBAAiB,IAAI;IAAC;IAGjD,OAAOzC;AACT;AAEO,eAAe0C,yBACpB7C,QAAsC,EACtCvB,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAMoE,WAAW,MAAM/C,OAAOC,UAAUvB,mBAAmBC;IAC3D0D,OAAOU,UAAU;IACjB,MAAM3D,SAAST,YAAY,MAAM;IACjC,MAAMqE,cAAcC,cAAcF,SAAS,OAAO,EAAE3D;IACpD,OAAO;QAAE,SAAS4D;QAAa,OAAOD,SAAS,KAAK;IAAC;AACvD;AAEO,eAAeG,yBACpBC,IAAY,EACZzE,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM,EAAEoC,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,OAAOmD,MAAMzE,mBAAmBC;IACjE,OAAO;QAAEoC;QAASE;IAAM;AAC1B;AAEO,SAASmC,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASR,cAAcQ,KAAa,EAAErE,MAAgC;IAC3E,MAAMsE,kBAAkBN,yBAAyBK;IAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAACvC;IACT;IACA,IAAI;QACF,OAAOgB,KAAK,KAAK,CAACsB;IACpB,EAAE,OAAM,CAAC;IACT,IAAI;QACF,OAAOtB,KAAK,KAAK,CAACwB,WAAWF;IAC/B,EAAE,OAAOpB,GAAG,CAAC;IAEb,IAAIlD,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAMyE,aAAaL,yBAAyBE;QAC5C,OAAOtB,KAAK,KAAK,CAACwB,WAAWC;IAC/B;IACA,MAAMpB,MAAM,CAAC,+BAA+B,EAAEgB,OAAO;AACvD"}

package/dist/es/device/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"device/index.mjs","sources":["webpack://@midscene/core/./src/device/index.ts"],"sourcesContent":["import { getMidsceneLocationSchema } from '@/ai-model';\nimport type { DeviceAction, LocateResultElement } from '@/types';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;\n\n  abstract cacheFeatureForRect?(\n    rect: Rect,\n    opt?: { _orderSensitive: boolean },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  // @deprecated do NOT extend this method\n  abstract getContext?(): Promise<UIContext>;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\nexport const defineAction = <\n  TSchema extends z.ZodType,\n  TRuntime = z.infer<TSchema>,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema: TSchema;\n    call: (param: TRuntime) => Promise<void>;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\n// Override the inferred type to use LocateResultElement for the runtime locate field\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    call,\n  });\n};\n\n// Input\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be input')\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'append'])\n    .default('replace')\n    .optional()\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"append\" - append the value to existing content; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    call,\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe('The direction to scroll'),\n  scrollType: z\n    .enum(['once', 'untilBottom', 'untilTop', 'untilRight', 'untilLeft'])\n    .default('once')\n    .describe('The scroll type'),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe('The element to be scrolled'),\n});\nexport type ActionScrollParam = {\n  direction?: 'down' | 'up' | 'right' | 'left';\n  scrollType?: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';\n  distance?: number | null;\n  locate?: LocateResultElement;\n};\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description: 'Drag and drop the element',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a swipe gesture. You must specify either \"end\" (target location) or \"distance\" + \"direction\" - they are mutually exclusive. Use \"end\" for precise location-based swipes, or \"distance\" + \"direction\" for relative movement.',\n    paramSchema: ActionSwipeParamSchema,\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The input field to be cleared'),\n});\nexport type ActionClearInputParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: 'Clear the text content of an input field',\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    call,\n  });\n};\n\nexport type { DeviceAction } from '../types';\n"],"names":["AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","actionInputParamSchema","val","String","defineActionInput","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput"],"mappings":";;AAOO,MAAeA;AAgCtB;AAIO,MAAMC,eAAe,CAI1BC,SAaOA;AAIF,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAMO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACbI;IACF;AAIK,MAAMC,8BAA8BJ,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACbD;IACF;AAIK,MAAMG,+BAA+BN,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACbH;IACF;AAIK,MAAMK,yBAAyBR,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACbL;IACF;AAIK,MAAMO,yBAAyBV,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACW,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQV,4BACL,QAAQ,CAAC,2BACT,QAAQ;IACX,MAAMD,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAS,EACnC,OAAO,CAAC,WACR,QAAQ,GACR,QAAQ,CACP;AAEN;AAOO,MAAMa,oBAAoB,CAC/BV,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaa;QACbP;IACF;AAIK,MAAMW,iCAAiCd,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMe,4BAA4B,CACvCZ,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaiB;QACbX;IACF;AAIK,MAAMa,0BAA0BhB,EAAE,MAAM,CAAC;IAC9C,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,YAAYA,CAAC,CAADA,OACL,CAAC;QAAC;QAAQ;QAAe;QAAY;QAAc;KAAY,EACnE,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CAAC;AACd;AAQO,MAAMgB,qBAAqB,CAChCd,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACbb;IACF;AAIK,MAAMe,+BAA+BlB,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMkB,0BAA0B,CACrChB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaqB;QACbf;IACF;AAGK,MAAMiB,6BAA6BpB,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMqB,wBAAwB,CACnClB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAauB;QACbjB;IACF;AAGK,MAAMmB,yBAAyBtB,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,MAAMuB,oBAAoB,CAC/BpB,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAayB;QACbnB;IACF;AAIK,MAAMqB,8BAA8BxB,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMwB,yBAAyB,CACpCtB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAa2B;QACbrB;IACF"}
+{"version":3,"file":"device/index.mjs","sources":["webpack://@midscene/core/./src/device/index.ts"],"sourcesContent":["import { getMidsceneLocationSchema } from '@/ai-model';\nimport type { DeviceAction, LocateResultElement } from '@/types';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;\n\n  abstract cacheFeatureForRect?(\n    rect: Rect,\n    opt?: { _orderSensitive: boolean },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  // @deprecated do NOT extend this method\n  abstract getContext?(): Promise<UIContext>;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\nexport const defineAction = <\n  TSchema extends z.ZodType,\n  TRuntime = z.infer<TSchema>,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema: TSchema;\n    call: (param: TRuntime) => Promise<void>;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\n// Override the inferred type to use LocateResultElement for the runtime locate field\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    call,\n  });\n};\n\n// Input\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be input')\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'append'])\n    .default('replace')\n    .optional()\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"append\" - append the value to existing content; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    call,\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe('The direction to scroll'),\n  scrollType: z\n    .enum(['once', 'untilBottom', 'untilTop', 'untilRight', 'untilLeft'])\n    .default('once')\n    .describe('The scroll type'),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe('The element to be scrolled'),\n});\nexport type ActionScrollParam = {\n  direction?: 'down' | 'up' | 'right' | 'left';\n  scrollType?: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';\n  distance?: number | null;\n  locate?: LocateResultElement;\n};\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description: 'Drag and drop the element',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a swipe gesture. You must specify either \"end\" (target location) or \"distance\" + \"direction\" - they are mutually exclusive. Use \"end\" for precise location-based swipes, or \"distance\" + \"direction\" for relative movement.',\n    paramSchema: ActionSwipeParamSchema,\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The input field to be cleared'),\n});\nexport type ActionClearInputParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: 'Clear the text content of an input field',\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    call,\n  });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n  AndroidDeviceOpt,\n  AndroidDeviceInputOpt,\n  IOSDeviceOpt,\n  IOSDeviceInputOpt,\n} from './device-options';\n"],"names":["AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","actionInputParamSchema","val","String","defineActionInput","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput"],"mappings":";;AAOO,MAAeA;AAgCtB;AAIO,MAAMC,eAAe,CAI1BC,SAaOA;AAIF,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAMO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACbI;IACF;AAIK,MAAMC,8BAA8BJ,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACbD;IACF;AAIK,MAAMG,+BAA+BN,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACbH;IACF;AAIK,MAAMK,yBAAyBR,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACbL;IACF;AAIK,MAAMO,yBAAyBV,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACW,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQV,4BACL,QAAQ,CAAC,2BACT,QAAQ;IACX,MAAMD,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAS,EACnC,OAAO,CAAC,WACR,QAAQ,GACR,QAAQ,CACP;AAEN;AAOO,MAAMa,oBAAoB,CAC/BV,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaa;QACbP;IACF;AAIK,MAAMW,iCAAiCd,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMe,4BAA4B,CACvCZ,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaiB;QACbX;IACF;AAIK,MAAMa,0BAA0BhB,EAAE,MAAM,CAAC;IAC9C,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,YAAYA,CAAC,CAADA,OACL,CAAC;QAAC;QAAQ;QAAe;QAAY;QAAc;KAAY,EACnE,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CAAC;AACd;AAQO,MAAMgB,qBAAqB,CAChCd,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACbb;IACF;AAIK,MAAMe,+BAA+BlB,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMkB,0BAA0B,CACrChB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaqB;QACbf;IACF;AAGK,MAAMiB,6BAA6BpB,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMqB,wBAAwB,CACnClB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAauB;QACbjB;IACF;AAGK,MAAMmB,yBAAyBtB,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,MAAMuB,oBAAoB,CAC/BpB,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAayB;QACbnB;IACF;AAIK,MAAMqB,8BAA8BxB,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMwB,yBAAyB,CACpCtB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAa2B;QACbrB;IACF"}

package/dist/es/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"types.mjs","sources":["webpack://@midscene/core/./src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type {\n  CreateOpenAIClientFn,\n  TModelConfigFn,\n} from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './ai-model/common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\nexport type { LocateResultElement };\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  abstract screenshotBase64: string;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type InsightExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface InsightTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface InsightDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: InsightExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: InsightTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialInsightDumpFromSDK = Omit<\n  InsightDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface InsightResultBase {\n  dump: InsightDump;\n}\n\nexport type LocateResultWithDump = LocateResult & InsightResultBase;\n\nexport interface InsightExtractResult<T> extends InsightResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n}\n\nexport class InsightError extends Error {\n  dump: InsightDump;\n\n  constructor(message: string, dump: InsightDump) {\n    super(message);\n    this.name = 'InsightError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type InsightAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  subTask?: boolean;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActionContext?: string;\n}\n\n/*\ntask - insight-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = InsightDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - insight-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: InsightExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  InsightAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<T = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<T>;\n  call: (param: T, context: ExecutorContext) => Promise<void> | void;\n}\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext;\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfigFn;\n  cache?: Cache;\n  replanningCycleLimit?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: (config) => {\n   *   const openai = new OpenAI({\n   *     apiKey: config.openaiApiKey,\n   *     baseURL: config.openaiBaseURL,\n   *   });\n   *\n   *   // Wrap with langsmith for planning tasks\n   *   if (config.intent === 'planning') {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["AIResponseFormat","UIContext","InsightError","Error","message","dump"],"mappings":";AAAqD;;;;;;;;;;AA0C9C,IAAKA,yBAAgBA,WAAAA,GAAAA,SAAhBA,gBAAgB;;;WAAhBA;;AA2EL,MAAeC;AAMtB;AAsEO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF"}
+{"version":3,"file":"types.mjs","sources":["webpack://@midscene/core/./src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type {\n  CreateOpenAIClientFn,\n  TModelConfigFn,\n} from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './ai-model/common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\nexport type { LocateResultElement };\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  abstract screenshotBase64: string;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type InsightExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface InsightTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface InsightDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: InsightExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: InsightTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialInsightDumpFromSDK = Omit<\n  InsightDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface InsightResultBase {\n  dump: InsightDump;\n}\n\nexport type LocateResultWithDump = LocateResult & InsightResultBase;\n\nexport interface InsightExtractResult<T> extends InsightResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n}\n\nexport class InsightError extends Error {\n  dump: InsightDump;\n\n  constructor(message: string, dump: InsightDump) {\n    super(message);\n    this.name = 'InsightError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type InsightAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  subTask?: boolean;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActionContext?: string;\n}\n\n/*\ntask - insight-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = InsightDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - insight-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: InsightExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  InsightAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<T = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<T>;\n  call: (param: T, context: ExecutorContext) => Promise<void> | void;\n}\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext;\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfigFn;\n  cache?: Cache;\n  replanningCycleLimit?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["AIResponseFormat","UIContext","InsightError","Error","message","dump"],"mappings":";AAAqD;;;;;;;;;;AA0C9C,IAAKA,yBAAgBA,WAAAA,GAAAA,SAAhBA,gBAAgB;;;WAAhBA;;AA2EL,MAAeC;AAMtB;AAsEO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF"}