@midscene/core 0.28.12-beta-20250923124135.0 → 0.28.12-beta-20250924031347.0

package/dist/lib/ai-model/ui-tars-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/ui-tars-planning.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/ui-tars-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIUsageInfo,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Size,\n} from '@/types';\nimport { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport { actionParser } from '@ui-tars/action-parser';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { AIActionType } from './common';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt/ui-tars-planning';\nimport { callAIWithStringResponse } from './service-caller/index';\ntype ActionType =\n  | 'click'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\nconst bboxSize = 10;\nconst pointToBbox = (\n  point: { x: number; y: number },\n  width: number,\n  height: number,\n): [number, number, number, number] => {\n  return [\n    Math.round(Math.max(point.x - bboxSize / 2, 0)),\n    Math.round(Math.max(point.y - bboxSize / 2, 0)),\n    Math.round(Math.min(point.x + bboxSize / 2, width)),\n    Math.round(Math.min(point.y + bboxSize / 2, height)),\n  ];\n};\n\nexport async function vlmPlanning(options: {\n  userInstruction: string;\n  conversationHistory: ChatCompletionMessageParam[];\n  size: { width: number; height: number };\n  modelConfig: IModelConfig;\n}): Promise<{\n  actions: PlanningAction<any>[];\n  actionsFromModel: ReturnType<typeof actionParser>['parsed'];\n  action_summary: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const { conversationHistory, userInstruction, size, modelConfig } = options;\n  const { uiTarsModelVersion } = modelConfig;\n  const systemPrompt = getUiTarsPlanningPrompt() + userInstruction;\n\n  const res = await callAIWithStringResponse(\n    [\n      {\n        role: 'user',\n        content: systemPrompt,\n      },\n      ...conversationHistory,\n    ],\n    AIActionType.INSPECT_ELEMENT,\n    modelConfig,\n  );\n  const convertedText = convertBboxToCoordinates(res.content);\n\n  const { parsed } = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: size.width,\n      height: size.height,\n    },\n    modelVer: uiTarsModelVersion,\n  });\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parsed),\n  );\n\n  const transformActions: PlanningAction[] = [];\n  parsed.forEach((action) => {\n    if (action.action_type === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: point[0], y: point[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n      });\n    } else if (action.action_type === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box, size);\n      const endPoint = getPoint(action.action_inputs.end_box, size);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: startPoint[0], y: startPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n          to: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: endPoint[0], y: endPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'finished') {\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'hotkey') {\n      if (!action.action_inputs.key) {\n        console.warn(\n          'No key found in action: hotkey. Will not perform action.',\n        );\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys,\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (action.action_type === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    }\n  });\n\n  if (transformActions.length === 0) {\n    throw new Error(`No actions found, response: ${res.content}`, {\n      cause: {\n        prediction: res.content,\n        parsed,\n      },\n    });\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n\n  return {\n    actions: transformActions,\n    actionsFromModel: parsed,\n    action_summary: getSummary(res.content),\n    usage: res.usage,\n    rawResponse: JSON.stringify(res.content, undefined, 2),\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points\n * @param text - The text containing bbox tags to be converted\n * @returns The text with bbox tags replaced by coordinate points\n */\nfunction convertBboxToCoordinates(text: string): string {\n  // Match the four numbers after <bbox>\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    // Convert strings to numbers and calculate center point\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    // Use Math.floor to truncate and calculate center point\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    // Return formatted coordinate string\n    return `(${x},${y})`;\n  }\n\n  // Remove [EOS] and replace <bbox> coordinates\n  const cleanedText = text.replace(/\\[EOS\\]/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\nfunction getPoint(startBox: string, size: { width: number; height: number }) {\n  const [x, y] = JSON.parse(startBox);\n  return [x * size.width, y * size.height];\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n    end_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: Record<string, never>;\n}\n\nexport type Action =\n  | ClickAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n\nexport async function resizeImageForUiTars(\n  imageBase64: string,\n  size: Size,\n  uiTarsVersion: UITarsModelVersion | undefined,\n) {\n  if (uiTarsVersion === UITarsModelVersion.V1_5) {\n    debug('ui-tars-v1.5, will check image size', size);\n    const currentPixels = size.width * size.height;\n    const maxPixels = 16384 * 28 * 28; //\n    if (currentPixels > maxPixels) {\n      const resizeFactor = Math.sqrt(maxPixels / currentPixels);\n      const newWidth = Math.floor(size.width * resizeFactor);\n      const newHeight = Math.floor(size.height * resizeFactor);\n      debug(\n        'resize image for ui-tars, new width: %s, new height: %s',\n        newWidth,\n        newHeight,\n      );\n      const resizedImage = await resizeImgBase64(imageBase64, {\n        width: newWidth,\n        height: newHeight,\n      });\n      return resizedImage;\n    }\n  }\n  return imageBase64;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","bboxSize","pointToBbox","point","width","height","Math","vlmPlanning","options","conversationHistory","userInstruction","size","modelConfig","uiTarsModelVersion","systemPrompt","getUiTarsPlanningPrompt","res","callAIWithStringResponse","AIActionType","convertedText","convertBboxToCoordinates","parsed","actionParser","JSON","transformActions","action","assert","getPoint","startPoint","endPoint","keys","transformHotkeyInput","console","Error","getSummary","undefined","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","y","cleanedText","startBox","resizeImageForUiTars","imageBase64","uiTarsVersion","UITarsModelVersion","currentPixels","maxPixels","resizeFactor","newWidth","newHeight","resizedImage","resizeImgBase64"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACmBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,WAAW;AACjB,MAAMC,cAAc,CAClBC,OACAC,OACAC,SAEO;QACLC,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGG;QAC5CE,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGI;KAC7C;AAGI,eAAeE,YAAYC,OAKjC;IAQC,MAAM,EAAEC,mBAAmB,EAAEC,eAAe,EAAEC,IAAI,EAAEC,WAAW,EAAE,GAAGJ;IACpE,MAAM,EAAEK,kBAAkB,EAAE,GAAGD;IAC/B,MAAME,eAAeC,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,MAA4BL;IAEjD,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChB;QACE;YACE,MAAM;YACN,SAASH;QACX;WACGL;KACJ,EACDS,mCAAAA,YAAAA,CAAAA,eAA4B,EAC5BN;IAEF,MAAMO,gBAAgBC,yBAAyBJ,IAAI,OAAO;IAE1D,MAAM,EAAEK,MAAM,EAAE,GAAGC,AAAAA,IAAAA,8BAAAA,YAAAA,AAAAA,EAAa;QAC9B,YAAYH;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOR,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACA,UAAUE;IACZ;IAEAd,MACE,oBACAc,oBACA,YACAU,KAAK,SAAS,CAACF;IAGjB,MAAMG,mBAAqC,EAAE;IAC7CH,OAAO,OAAO,CAAC,CAACI;QACd,IAAIA,AAAuB,YAAvBA,OAAO,WAAW,EAAc;YAClCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMtB,QAAQwB,SAASF,OAAO,aAAa,CAAC,SAAS,EAAEd;YACvDa,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQ;wBACN,QAAQC,OAAO,OAAO,IAAI;wBAC1B,MAAMvB,YACJ;4BAAE,GAAGC,KAAK,CAAC,EAAE;4BAAE,GAAGA,KAAK,CAAC,EAAE;wBAAC,GAC3BQ,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;YACF;QACF,OAAO,IAAIc,AAAuB,WAAvBA,OAAO,WAAW,EAAa;YACxCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMG,aAAaD,SAASF,OAAO,aAAa,CAAC,SAAS,EAAEd;YAC5D,MAAMkB,WAAWF,SAASF,OAAO,aAAa,CAAC,OAAO,EAAEd;YACxDa,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,QAAQC,OAAO,OAAO,IAAI;wBAC1B,MAAMvB,YACJ;4BAAE,GAAG0B,UAAU,CAAC,EAAE;4BAAE,GAAGA,UAAU,CAAC,EAAE;wBAAC,GACrCjB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;oBACA,IAAI;wBACF,QAAQc,OAAO,OAAO,IAAI;wBAC1B,MAAMvB,YACJ;4BAAE,GAAG2B,QAAQ,CAAC,EAAE;4BAAE,GAAGA,QAAQ,CAAC,EAAE;wBAAC,GACjClB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASc,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIA,AAAuB,WAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOC,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,aAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWC,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,eAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO,CAAC;YACR,SAASC,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,aAAvBA,OAAO,WAAW,EAC3B,IAAKA,OAAO,aAAa,CAAC,GAAG,EAItB;YACL,MAAMK,OAAOC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBN,OAAO,aAAa,CAAC,GAAG;YAE1DD,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASM;gBACX;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAbEO,QAAQ,IAAI,CACV;aAaC,IAAIP,AAAuB,WAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASC,OAAO,OAAO,IAAI;QAC7B;IAEJ;IAEA,IAAID,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIS,MAAM,CAAC,4BAA4B,EAAEjB,IAAI,OAAO,EAAE,EAAE;QAC5D,OAAO;YACL,YAAYA,IAAI,OAAO;YACvBK;QACF;IACF;IAGFtB,MAAM,oBAAoBwB,KAAK,SAAS,CAACC,kBAAkB,MAAM;IAEjE,OAAO;QACL,SAASA;QACT,kBAAkBH;QAClB,gBAAgBa,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWlB,IAAI,OAAO;QACtC,OAAOA,IAAI,KAAK;QAChB,aAAaO,KAAK,SAAS,CAACP,IAAI,OAAO,EAAEmB,QAAW;IACtD;AACF;AAOA,SAASf,yBAAyBgB,IAAY;IAE5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAGV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAGlC,MAAMM,IAAI3C,KAAK,KAAK,CAAEsC,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMG,IAAI5C,KAAK,KAAK,CAAEwC,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAGvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEC,EAAE,CAAC,CAAC;IACtB;IAGA,MAAMC,cAAcf,KAAK,OAAO,CAAC,YAAY;IAC7C,OAAOe,YAAY,OAAO,CAACd,SAASC,cAAc,IAAI;AACxD;AAEA,SAASX,SAASyB,QAAgB,EAAEzC,IAAuC;IACzE,MAAM,CAACsC,GAAGC,EAAE,GAAG3B,KAAK,KAAK,CAAC6B;IAC1B,OAAO;QAACH,IAAItC,KAAK,KAAK;QAAEuC,IAAIvC,KAAK,MAAM;KAAC;AAC1C;AAkEO,eAAe0C,qBACpBC,WAAmB,EACnB3C,IAAU,EACV4C,aAA6C;IAE7C,IAAIA,kBAAkBC,oBAAAA,kBAAAA,CAAAA,IAAuB,EAAE;QAC7CzD,MAAM,uCAAuCY;QAC7C,MAAM8C,gBAAgB9C,KAAK,KAAK,GAAGA,KAAK,MAAM;QAC9C,MAAM+C,YAAY;QAClB,IAAID,gBAAgBC,WAAW;YAC7B,MAAMC,eAAerD,KAAK,IAAI,CAACoD,YAAYD;YAC3C,MAAMG,WAAWtD,KAAK,KAAK,CAACK,KAAK,KAAK,GAAGgD;YACzC,MAAME,YAAYvD,KAAK,KAAK,CAACK,KAAK,MAAM,GAAGgD;YAC3C5D,MACE,2DACA6D,UACAC;YAEF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBT,aAAa;gBACtD,OAAOM;gBACP,QAAQC;YACV;YACA,OAAOC;QACT;IACF;IACA,OAAOR;AACT"}
+{"version":3,"file":"ai-model/ui-tars-planning.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/ui-tars-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  PlanningAIResponse,\n  PlanningAction,\n  Size,\n  UIContext,\n} from '@/types';\nimport { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport { actionParser } from '@ui-tars/action-parser';\nimport { AIActionType } from './common';\nimport type { ConversationHistory } from './conversation-history';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt/ui-tars-planning';\nimport { callAIWithStringResponse } from './service-caller/index';\ntype ActionType =\n  | 'click'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\nconst bboxSize = 10;\nconst pointToBbox = (\n  point: { x: number; y: number },\n  width: number,\n  height: number,\n): [number, number, number, number] => {\n  return [\n    Math.round(Math.max(point.x - bboxSize / 2, 0)),\n    Math.round(Math.max(point.y - bboxSize / 2, 0)),\n    Math.round(Math.min(point.x + bboxSize / 2, width)),\n    Math.round(Math.min(point.y + bboxSize / 2, height)),\n  ];\n};\n\nexport async function uiTarsPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig } = options;\n  const { uiTarsModelVersion } = modelConfig;\n  const systemPrompt = getUiTarsPlanningPrompt() + userInstruction;\n\n  const imagePayload = await resizeImageForUiTars(\n    context.screenshotBase64,\n    context.size,\n    uiTarsModelVersion,\n  );\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: {\n          url: imagePayload,\n        },\n      },\n    ],\n  });\n\n  const res = await callAIWithStringResponse(\n    [\n      {\n        role: 'user',\n        content: systemPrompt,\n      },\n      ...conversationHistory.snapshot(),\n    ],\n    AIActionType.INSPECT_ELEMENT,\n    modelConfig,\n  );\n  const convertedText = convertBboxToCoordinates(res.content);\n\n  const { size } = context;\n  const { parsed } = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: size.width,\n      height: size.height,\n    },\n    modelVer: uiTarsModelVersion,\n  });\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parsed),\n  );\n\n  const transformActions: PlanningAction[] = [];\n  let shouldContinue = true;\n  parsed.forEach((action) => {\n    const actionType = (action.action_type || '').toLowerCase();\n    if (actionType === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: point[0], y: point[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n      });\n    } else if (actionType === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box, size);\n      const endPoint = getPoint(action.action_inputs.end_box, size);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: startPoint[0], y: startPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n          to: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: endPoint[0], y: endPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'finished') {\n      shouldContinue = false;\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'hotkey') {\n      if (!action.action_inputs.key) {\n        console.warn(\n          'No key found in action: hotkey. Will not perform action.',\n        );\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys,\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (actionType === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    }\n  });\n\n  if (transformActions.length === 0) {\n    throw new Error(`No actions found, response: ${res.content}`, {\n      cause: {\n        prediction: res.content,\n        parsed,\n      },\n    });\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n  const log = getSummary(res.content);\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: log,\n  });\n\n  return {\n    actions: transformActions,\n    log,\n    usage: res.usage,\n    rawResponse: JSON.stringify(res.content, undefined, 2),\n    more_actions_needed_by_instruction: shouldContinue,\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points\n * @param text - The text containing bbox tags to be converted\n * @returns The text with bbox tags replaced by coordinate points\n */\nfunction convertBboxToCoordinates(text: string): string {\n  // Match the four numbers after <bbox>\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    // Convert strings to numbers and calculate center point\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    // Use Math.floor to truncate and calculate center point\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    // Return formatted coordinate string\n    return `(${x},${y})`;\n  }\n\n  // Remove [EOS] and replace <bbox> coordinates\n  const cleanedText = text.replace(/\\[EOS\\]/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\nfunction getPoint(startBox: string, size: { width: number; height: number }) {\n  const [x, y] = JSON.parse(startBox);\n  return [x * size.width, y * size.height];\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n    end_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: Record<string, never>;\n}\n\nexport type Action =\n  | ClickAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n\nexport async function resizeImageForUiTars(\n  imageBase64: string,\n  size: Size,\n  uiTarsVersion: UITarsModelVersion | undefined,\n) {\n  if (uiTarsVersion === UITarsModelVersion.V1_5) {\n    debug('ui-tars-v1.5, will check image size', size);\n    const currentPixels = size.width * size.height;\n    const maxPixels = 16384 * 28 * 28; //\n    if (currentPixels > maxPixels) {\n      const resizeFactor = Math.sqrt(maxPixels / currentPixels);\n      const newWidth = Math.floor(size.width * resizeFactor);\n      const newHeight = Math.floor(size.height * resizeFactor);\n      debug(\n        'resize image for ui-tars, new width: %s, new height: %s',\n        newWidth,\n        newHeight,\n      );\n      const resizedImage = await resizeImgBase64(imageBase64, {\n        width: newWidth,\n        height: newHeight,\n      });\n      return resizedImage;\n    }\n  }\n  return imageBase64;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","bboxSize","pointToBbox","point","width","height","Math","uiTarsPlanning","userInstruction","options","conversationHistory","context","modelConfig","uiTarsModelVersion","systemPrompt","getUiTarsPlanningPrompt","imagePayload","resizeImageForUiTars","res","callAIWithStringResponse","AIActionType","convertedText","convertBboxToCoordinates","size","parsed","actionParser","JSON","transformActions","shouldContinue","action","actionType","assert","getPoint","startPoint","endPoint","keys","transformHotkeyInput","console","Error","log","getSummary","undefined","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","y","cleanedText","startBox","imageBase64","uiTarsVersion","UITarsModelVersion","currentPixels","maxPixels","resizeFactor","newWidth","newHeight","resizedImage","resizeImgBase64"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACmBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,WAAW;AACjB,MAAMC,cAAc,CAClBC,OACAC,OACAC,SAEO;QACLC,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGG;QAC5CE,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGI;KAC7C;AAGI,eAAeE,eACpBC,eAAuB,EACvBC,OAIC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGH;IACtD,MAAM,EAAEI,kBAAkB,EAAE,GAAGD;IAC/B,MAAME,eAAeC,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,MAA4BP;IAEjD,MAAMQ,eAAe,MAAMC,qBACzBN,QAAQ,gBAAgB,EACxBA,QAAQ,IAAI,EACZE;IAGFH,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKM;gBACP;YACF;SACD;IACH;IAEA,MAAME,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChB;QACE;YACE,MAAM;YACN,SAASL;QACX;WACGJ,oBAAoB,QAAQ;KAChC,EACDU,mCAAAA,YAAAA,CAAAA,eAA4B,EAC5BR;IAEF,MAAMS,gBAAgBC,yBAAyBJ,IAAI,OAAO;IAE1D,MAAM,EAAEK,IAAI,EAAE,GAAGZ;IACjB,MAAM,EAAEa,MAAM,EAAE,GAAGC,AAAAA,IAAAA,8BAAAA,YAAAA,AAAAA,EAAa;QAC9B,YAAYJ;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOE,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACA,UAAUV;IACZ;IAEAd,MACE,oBACAc,oBACA,YACAa,KAAK,SAAS,CAACF;IAGjB,MAAMG,mBAAqC,EAAE;IAC7C,IAAIC,iBAAiB;IACrBJ,OAAO,OAAO,CAAC,CAACK;QACd,MAAMC,aAAcD,AAAAA,CAAAA,OAAO,WAAW,IAAI,EAAC,EAAG,WAAW;QACzD,IAAIC,AAAe,YAAfA,YAAwB;YAC1BC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAM1B,QAAQ6B,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEN;YACvDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQ;wBACN,QAAQE,OAAO,OAAO,IAAI;wBAC1B,MAAM3B,YACJ;4BAAE,GAAGC,KAAK,CAAC,EAAE;4BAAE,GAAGA,KAAK,CAAC,EAAE;wBAAC,GAC3BoB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;YACF;QACF,OAAO,IAAIO,AAAe,WAAfA,YAAuB;YAChCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMI,aAAaD,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEN;YAC5D,MAAMW,WAAWF,SAASH,OAAO,aAAa,CAAC,OAAO,EAAEN;YACxDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,QAAQE,OAAO,OAAO,IAAI;wBAC1B,MAAM3B,YACJ;4BAAE,GAAG+B,UAAU,CAAC,EAAE;4BAAE,GAAGA,UAAU,CAAC,EAAE;wBAAC,GACrCV,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;oBACA,IAAI;wBACF,QAAQM,OAAO,OAAO,IAAI;wBAC1B,MAAM3B,YACJ;4BAAE,GAAGgC,QAAQ,CAAC,EAAE;4BAAE,GAAGA,QAAQ,CAAC,EAAE;wBAAC,GACjCX,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASM,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOE,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,aAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWE,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,eAAfA,YAA2B;YACpCF,iBAAiB;YACjBD,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO,CAAC;gBACR,SAASE,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,aAAfA,YACT,IAAKD,OAAO,aAAa,CAAC,GAAG,EAItB;YACL,MAAMM,OAAOC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBP,OAAO,aAAa,CAAC,GAAG;YAE1DF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASQ;gBACX;gBACA,SAASN,OAAO,OAAO,IAAI;YAC7B;QACF,OAbEQ,QAAQ,IAAI,CACV;aAaC,IAAIP,AAAe,WAAfA,YACTH,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASE,OAAO,OAAO,IAAI;QAC7B;IAEJ;IAEA,IAAIF,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIW,MAAM,CAAC,4BAA4B,EAAEpB,IAAI,OAAO,EAAE,EAAE;QAC5D,OAAO;YACL,YAAYA,IAAI,OAAO;YACvBM;QACF;IACF;IAGFzB,MAAM,oBAAoB2B,KAAK,SAAS,CAACC,kBAAkB,MAAM;IACjE,MAAMY,MAAMC,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWtB,IAAI,OAAO;IAElCR,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS6B;IACX;IAEA,OAAO;QACL,SAASZ;QACTY;QACA,OAAOrB,IAAI,KAAK;QAChB,aAAaQ,KAAK,SAAS,CAACR,IAAI,OAAO,EAAEuB,QAAW;QACpD,oCAAoCb;IACtC;AACF;AAOA,SAASN,yBAAyBoB,IAAY;IAE5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAGV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAGlC,MAAMM,IAAIjD,KAAK,KAAK,CAAE4C,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMG,IAAIlD,KAAK,KAAK,CAAE8C,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAGvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEC,EAAE,CAAC,CAAC;IACtB;IAGA,MAAMC,cAAcf,KAAK,OAAO,CAAC,YAAY;IAC7C,OAAOe,YAAY,OAAO,CAACd,SAASC,cAAc,IAAI;AACxD;AAEA,SAASZ,SAAS0B,QAAgB,EAAEnC,IAAuC;IACzE,MAAM,CAACgC,GAAGC,EAAE,GAAG9B,KAAK,KAAK,CAACgC;IAC1B,OAAO;QAACH,IAAIhC,KAAK,KAAK;QAAEiC,IAAIjC,KAAK,MAAM;KAAC;AAC1C;AAkEO,eAAeN,qBACpB0C,WAAmB,EACnBpC,IAAU,EACVqC,aAA6C;IAE7C,IAAIA,kBAAkBC,oBAAAA,kBAAAA,CAAAA,IAAuB,EAAE;QAC7C9D,MAAM,uCAAuCwB;QAC7C,MAAMuC,gBAAgBvC,KAAK,KAAK,GAAGA,KAAK,MAAM;QAC9C,MAAMwC,YAAY;QAClB,IAAID,gBAAgBC,WAAW;YAC7B,MAAMC,eAAe1D,KAAK,IAAI,CAACyD,YAAYD;YAC3C,MAAMG,WAAW3D,KAAK,KAAK,CAACiB,KAAK,KAAK,GAAGyC;YACzC,MAAME,YAAY5D,KAAK,KAAK,CAACiB,KAAK,MAAM,GAAGyC;YAC3CjE,MACE,2DACAkE,UACAC;YAEF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBT,aAAa;gBACtD,OAAOM;gBACP,QAAQC;YACV;YACA,OAAOC;QACT;IACF;IACA,OAAOR;AACT"}

package/dist/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/index.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { z } from 'zod';\nimport { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  getMidsceneLocationSchema,\n  type MidsceneLocationResultType,\n  PointSchema,\n  SizeSchema,\n  RectSchema,\n  TMultimodalPromptSchema,\n  TUserPromptSchema,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n} from './ai-model/index';\n\nexport { MIDSCENE_MODEL_NAME } from '@midscene/shared/env';\n\nexport type * from './types';\n\nexport { z };\n\nexport default Insight;\nexport { Executor, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlConfigResult,\n  MidsceneYamlConfig,\n  MidsceneYamlScriptWebEnv,\n  MidsceneYamlScriptAndroidEnv,\n  MidsceneYamlScriptIOSEnv,\n  MidsceneYamlScriptEnv,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n\nexport { Agent, type AgentOpt, createAgent } from './agent';\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","Insight"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACoBA,YAAeI"}
+{"version":3,"file":"index.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/index.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { z } from 'zod';\nimport { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  getMidsceneLocationSchema,\n  type MidsceneLocationResultType,\n  PointSchema,\n  SizeSchema,\n  RectSchema,\n  TMultimodalPromptSchema,\n  TUserPromptSchema,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n} from './ai-model/index';\n\nexport { MIDSCENE_MODEL_NAME } from '@midscene/shared/env';\n\nexport type * from './types';\n\nexport { z };\n\nexport default Insight;\nexport { Executor, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlConfigResult,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n\nexport { Agent, type AgentOpt, createAgent } from './agent';\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","Insight"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACoBA,YAAeI"}

package/dist/lib/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/types.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { TModelConfigFn } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './ai-model/common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\nexport interface AIElementLocatorResponse {\n  elements: {\n    id: string;\n    reason?: string;\n    text?: string;\n    xpaths?: string[];\n  }[];\n  bbox?: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport type AIElementResponse =\n  | AIElementLocatorResponse\n  | AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext<ElementType extends BaseElement = BaseElement> {\n  abstract screenshotBase64: string;\n\n  abstract tree: ElementTreeNode<ElementType>;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type InsightExtractParam = string | Record<string, string>;\n\nexport type LocateResultElement = {\n  center: [number, number];\n  rect: Rect;\n  id: string;\n  indexId?: number;\n  xpaths: string[];\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n  isOrderSensitive?: boolean;\n};\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface InsightTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  sdkVersion: string;\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface InsightDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: InsightExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: BaseElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: InsightTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialInsightDumpFromSDK = Omit<\n  InsightDump,\n  'sdkVersion' | 'logTime' | 'logId' | 'model_name'\n>;\n\nexport type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type InsightAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  id?: string;\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface PlanningAIResponse {\n  action?: PlanningAction; // this is the qwen mode\n  actions?: PlanningAction[];\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport type PlanningActionParamTap = null;\nexport type PlanningActionParamHover = null;\nexport type PlanningActionParamRightClick = null;\n\nexport interface PlanningActionParamInputOrKeyPress {\n  value: string;\n  autoDismissKeyboard?: boolean;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface AndroidLongPressParam {\n  duration?: number;\n}\n\nexport interface AndroidPullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n}\n\n/*\ntask - insight-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskInsightDumpLog {\n  dump?: InsightDump;\n}\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - insight-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: InsightExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  InsightAssertionResponse,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    log?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<T = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<T>;\n  call: (param: T, context: ExecutorContext) => Promise<void> | void;\n}\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext<WebElementInfo>;\n\n/**\n * Agent\n */\n\nexport interface AgentOpt {\n  testId?: string;\n  cacheId?: string;\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfigFn;\n  useCache?: boolean;\n  replanningCycleLimit?: number;\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","AIResponseFormat","UIContext"],"mappings":";;;;;;;;;;;;;;;;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;ICgCO,IAAKI,mBAAAA,WAAAA,GAAAA,SAAAA,gBAAgB;;;eAAhBA;;IAwFL,MAAeC;IAQtB"}
+{"version":3,"file":"types.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/types.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { TModelConfigFn } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './ai-model/common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\nexport interface AIElementLocatorResponse {\n  elements: {\n    id: string;\n    reason?: string;\n    text?: string;\n    xpaths?: string[];\n  }[];\n  bbox?: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport type AIElementResponse =\n  | AIElementLocatorResponse\n  | AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext<ElementType extends BaseElement = BaseElement> {\n  abstract screenshotBase64: string;\n\n  abstract tree: ElementTreeNode<ElementType>;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type InsightExtractParam = string | Record<string, string>;\n\nexport type LocateResultElement = {\n  center: [number, number];\n  rect: Rect;\n  id: string;\n  indexId?: number;\n  xpaths: string[];\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n  isOrderSensitive?: boolean;\n};\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface InsightTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  sdkVersion: string;\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface InsightDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: InsightExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: BaseElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: InsightTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialInsightDumpFromSDK = Omit<\n  InsightDump,\n  'sdkVersion' | 'logTime' | 'logId' | 'model_name'\n>;\n\nexport type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type InsightAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  id?: string;\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface PlanningAIResponse {\n  action?: PlanningAction; // this is the qwen mode\n  actions?: PlanningAction[];\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport type PlanningActionParamTap = null;\nexport type PlanningActionParamHover = null;\nexport type PlanningActionParamRightClick = null;\n\nexport interface PlanningActionParamInputOrKeyPress {\n  value: string;\n  autoDismissKeyboard?: boolean;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface AndroidLongPressParam {\n  duration?: number;\n}\n\nexport interface AndroidPullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n}\n\n/*\ntask - insight-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskInsightDumpLog {\n  dump?: InsightDump;\n}\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - insight-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: InsightExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  InsightAssertionResponse,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<T = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<T>;\n  call: (param: T, context: ExecutorContext) => Promise<void> | void;\n}\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext<WebElementInfo>;\n\n/**\n * Agent\n */\n\nexport interface AgentOpt {\n  testId?: string;\n  cacheId?: string;\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfigFn;\n  useCache?: boolean;\n  replanningCycleLimit?: number;\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","AIResponseFormat","UIContext"],"mappings":";;;;;;;;;;;;;;;;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;ICgCO,IAAKI,mBAAAA,WAAAA,GAAAA,SAAAA,gBAAgB;;;eAAhBA;;IAwFL,MAAeC;IAQtB"}