@midscene/core 0.26.7-beta-20250818034910.0 → 0.26.7-beta-20250818035341.0

package/dist/es/ai-model/common.mjs

@@ -13,7 +13,6 @@ var common_AIActionType = /*#__PURE__*/ function(AIActionType) {
     AIActionType[AIActionType["DESCRIBE_ELEMENT"] = 4] = "DESCRIBE_ELEMENT";
     return AIActionType;
 }({});
-const actionSpaceTypePrefix = 'action_space_';
 async function callAiFn(msgs, AIActionTypeValue) {
     const jsonObject = await callToGetJSONObject(msgs, AIActionTypeValue);
     return {
@@ -170,29 +169,51 @@ async function markupImageForLLM(screenshotBase64, tree, size) {
     });
     return imagePayload;
 }
-function buildYamlFlowFromPlans(plans, actionSpace, sleep) {
+function buildYamlFlowFromPlans(plans, sleep) {
     const flow = [];
     for (const plan of plans){
         var _plan_locate;
-        const verb = plan.type;
-        const action = actionSpace.find((action)=>action.name === verb);
-        if (!action) {
-            console.warn(`Cannot convert action ${verb} to yaml flow. Will ignore it.`);
-            continue;
-        }
+        const type = plan.type;
         const locate = null == (_plan_locate = plan.locate) ? void 0 : _plan_locate.prompt;
-        const flowKey = action.interfaceAlias || `${actionSpaceTypePrefix}${verb}`;
-        const flowItem = {
-            [flowKey]: locate || '',
-            ...plan.param || {}
-        };
-        flow.push(flowItem);
+        if ('Tap' === type) flow.push({
+            aiTap: locate
+        });
+        else if ('Hover' === type) flow.push({
+            aiHover: locate
+        });
+        else if ('Input' === type) {
+            const param = plan.param;
+            flow.push({
+                aiInput: param.value,
+                locate
+            });
+        } else if ('KeyboardPress' === type) {
+            const param = plan.param;
+            flow.push({
+                aiKeyboardPress: param.value,
+                locate
+            });
+        } else if ('Scroll' === type) {
+            const param = plan.param;
+            flow.push({
+                aiScroll: null,
+                locate,
+                direction: param.direction,
+                scrollType: param.scrollType,
+                distance: param.distance
+            });
+        } else if ('Sleep' === type) {
+            const param = plan.param;
+            flow.push({
+                sleep: param.timeMs
+            });
+        } else 'AndroidBackButton' === type || 'AndroidHomeButton' === type || 'AndroidRecentAppsButton' === type || 'AndroidLongPress' === type || 'AndroidPull' === type || 'Error' === type || 'Assert' === type || 'AssertWithoutThrow' === type || 'Finished' === type || console.warn(`Cannot convert action ${type} to yaml flow. This should be a bug of Midscene.`);
     }
     if (sleep) flow.push({
-        sleep
+        sleep: sleep
     });
     return flow;
 }
-export { common_AIActionType as AIActionType, actionSpaceTypePrefix, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptQwenBbox, buildYamlFlowFromPlans, callAiFn, expandSearchArea, fillBboxParam, markupImageForLLM, mergeRects, warnGPT4oSizeLimit };
+export { common_AIActionType as AIActionType, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptQwenBbox, buildYamlFlowFromPlans, callAiFn, expandSearchArea, fillBboxParam, markupImageForLLM, mergeRects, warnGPT4oSizeLimit };
 
 //# sourceMappingURL=common.mjs.map

package/dist/es/ai-model/common.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport {\n  call,\n  callToGetJSONObject,\n  getModelName,\n} from './service-caller/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport { vlLocateMode } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n}\n\nexport const actionSpaceTypePrefix = 'action_space_';\n\nexport async function callAiFn<T>(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const jsonObject = await callToGetJSONObject<T>(msgs, AIActionTypeValue);\n\n  return {\n    content: jsonObject.content,\n    usage: jsonObject.usage,\n  };\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(locate.bbox, width, height);\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  if (vlLocateMode() === 'doubao-vision' || vlLocateMode() === 'vlm-ui-tars') {\n    return adaptDoubaoBbox(bbox, width, height);\n  }\n\n  if (vlLocateMode() === 'gemini') {\n    return adaptGeminiBbox(bbox, width, height);\n  }\n\n  return adaptQwenBbox(bbox);\n}\n\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n): Rect {\n  debugInspectUtils('adaptBboxToRect', bbox, width, height, offsetX, offsetY);\n  const [left, top, right, bottom] = adaptBbox(bbox, width, height);\n  const rect = {\n    left: left + offsetX,\n    top: top + offsetY,\n    width: right - left,\n    height: bottom - top,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(size: Size) {\n  if (warned) return;\n  if (getModelName()?.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your page to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(rect: Rect, screenSize: Size) {\n  const minEdgeSize = vlLocateMode() === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n  rect.left = Math.max(0, rect.left - paddingSizeHorizontal);\n  rect.width = Math.min(\n    rect.width + paddingSizeHorizontal * 2,\n    screenSize.width - rect.left,\n  );\n  rect.top = Math.max(0, rect.top - paddingSizeVertical);\n  rect.height = Math.min(\n    rect.height + paddingSizeVertical * 2,\n    screenSize.height - rect.top,\n  );\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const locate = plan.locate?.prompt;\n    const flowKey = action.interfaceAlias || `${actionSpaceTypePrefix}${verb}`;\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: locate || '',\n      ...(plan.param || {}),\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n"],"names":["AIActionType","actionSpaceTypePrefix","callAiFn","msgs","AIActionTypeValue","jsonObject","callToGetJSONObject","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","vlLocateMode","adaptGeminiBbox","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rect","warned","warnGPT4oSizeLimit","size","_getModelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","_plan_locate","verb","action","flowKey","flowItem"],"mappings":";;;;;;;AAkCO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;WAAZA;;AAQL,MAAMC,wBAAwB;AAE9B,eAAeC,SACpBC,IAAY,EACZC,iBAA+B;IAE/B,MAAMC,aAAa,MAAMC,oBAAuBH,MAAMC;IAEtD,OAAO;QACL,SAASC,WAAW,OAAO;QAC3B,OAAOA,WAAW,KAAK;IACzB;AACF;AAEA,MAAME,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc;IAGd,IAAKF,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGG,UAAUH,OAAO,IAAI,EAAEC,OAAOC;IAG9C,OAAOF;AACT;AAEO,SAASI,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;QACN,YAAnB,OAAOS,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;KAC1B;IACD,OAAOa;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCJ,KAAa,EACbC,MAAc;IAEdU,OACEX,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOG,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;YAC5CQ,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;SAC7C;QAEH,MAAM,IAAIM,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAIH,IACEc,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;QAEhEc,KAAK,GAAG,CACNT,OACAS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACNR,QACAQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIS,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAGH,MAAMI,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdJ,KAAa,EACbC,MAAc;IAEd,IAAIkB,AAAmB,oBAAnBA,kBAAsCA,AAAmB,kBAAnBA,gBACxC,OAAOT,gBAAgBN,MAAMJ,OAAOC;IAGtC,IAAIkB,AAAmB,aAAnBA,gBACF,OAAOC,gBAAgBhB,MAAMJ,OAAOC;IAGtC,OAAOE,cAAcC;AACvB;AAEO,SAASgB,gBACdhB,IAAc,EACdJ,KAAa,EACbC,MAAc;IAEd,MAAMoB,OAAOZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC5C,MAAMsB,MAAMb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC5C,MAAMsB,QAAQd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC7C,MAAMwB,SAASf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC/C,OAAO;QAACoB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdrB,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdyB,UAAU,CAAC,EACXC,UAAU,CAAC;IAEX/B,kBAAkB,mBAAmBQ,MAAMJ,OAAOC,QAAQyB,SAASC;IACnE,MAAM,CAACN,MAAMC,KAAKC,OAAOC,OAAO,GAAGtB,UAAUE,MAAMJ,OAAOC;IAC1D,MAAM2B,OAAO;QACX,MAAMP,OAAOK;QACb,KAAKJ,MAAMK;QACX,OAAOJ,QAAQF;QACf,QAAQG,SAASF;IACnB;IACA1B,kBAAkB,4BAA4BgC;IAC9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBAAmBC,IAAU;QAEvCC;IADJ,IAAIH,QAAQ;IACZ,IAAI,QAAAG,CAAAA,gBAAAA,cAAa,IAAbA,KAAAA,IAAAA,cAAgB,WAAW,GAAG,QAAQ,CAAC,WAAW;QACpD,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,wFAAwF,CAAC;QAEhN,IACEtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpCtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBAAiBd,IAAU,EAAEe,UAAgB;IAC3D,MAAMC,cAAczB,AAAmB,oBAAnBA,iBAAqC,MAAM;IAC/D,MAAM0B,iBAAiB;IAEvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IACNjB,KAAK,IAAI,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,IAAI,GAAGkB;IACpClB,KAAK,KAAK,GAAGnB,KAAK,GAAG,CACnBmB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA,uBACbH,WAAW,KAAK,GAAGf,KAAK,IAAI;IAE9BA,KAAK,GAAG,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,GAAG,GAAGmB;IAClCnB,KAAK,MAAM,GAAGnB,KAAK,GAAG,CACpBmB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA,qBACdJ,WAAW,MAAM,GAAGf,KAAK,GAAG;IAE9B,OAAOA;AACT;AAEO,eAAeoB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCnB,IAAU;IAEV,MAAMoB,eAAeC,WAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtBtB;IACF;IACA,OAAOyB;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAA2B,EAC3BC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;YAWTK;QAVf,MAAMC,OAAOF,KAAK,IAAI;QAEtB,MAAMG,SAASN,YAAY,IAAI,CAAC,CAACM,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXhC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAE+B,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMlE,SAAS,QAAAiE,CAAAA,eAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,MAAM;QAClC,MAAMG,UAAUD,OAAO,cAAc,IAAI,GAAG7E,wBAAwB4E,MAAM;QAE1E,MAAMG,WAAiC;YACrC,CAACD,QAAQ,EAAEpE,UAAU;YACrB,GAAIgE,KAAK,KAAK,IAAI,CAAC,CAAC;QACtB;QAEAD,KAAK,IAAI,CAACM;IACZ;IAEA,IAAIP,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT"}
+{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  PlanningActionParamInputOrKeyPress,\n  PlanningActionParamSleep,\n  Rect,\n  ScrollParam,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport {\n  call,\n  callToGetJSONObject,\n  getModelName,\n} from './service-caller/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport { vlLocateMode } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n}\n\nexport async function callAiFn<T>(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const jsonObject = await callToGetJSONObject<T>(msgs, AIActionTypeValue);\n\n  return {\n    content: jsonObject.content,\n    usage: jsonObject.usage,\n  };\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(locate.bbox, width, height);\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  if (vlLocateMode() === 'doubao-vision' || vlLocateMode() === 'vlm-ui-tars') {\n    return adaptDoubaoBbox(bbox, width, height);\n  }\n\n  if (vlLocateMode() === 'gemini') {\n    return adaptGeminiBbox(bbox, width, height);\n  }\n\n  return adaptQwenBbox(bbox);\n}\n\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n): Rect {\n  debugInspectUtils('adaptBboxToRect', bbox, width, height, offsetX, offsetY);\n  const [left, top, right, bottom] = adaptBbox(bbox, width, height);\n  const rect = {\n    left: left + offsetX,\n    top: top + offsetY,\n    width: right - left,\n    height: bottom - top,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(size: Size) {\n  if (warned) return;\n  if (getModelName()?.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your page to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(rect: Rect, screenSize: Size) {\n  const minEdgeSize = vlLocateMode() === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n  rect.left = Math.max(0, rect.left - paddingSizeHorizontal);\n  rect.width = Math.min(\n    rect.width + paddingSizeHorizontal * 2,\n    screenSize.width - rect.left,\n  );\n  rect.top = Math.max(0, rect.top - paddingSizeVertical);\n  rect.height = Math.min(\n    rect.height + paddingSizeVertical * 2,\n    screenSize.height - rect.top,\n  );\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const type = plan.type;\n    const locate = plan.locate?.prompt!; // TODO: check if locate is null\n\n    if (type === 'Tap') {\n      flow.push({\n        aiTap: locate!,\n      });\n    } else if (type === 'Hover') {\n      flow.push({\n        aiHover: locate!,\n      });\n    } else if (type === 'Input') {\n      const param = plan.param as PlanningActionParamInputOrKeyPress;\n      flow.push({\n        aiInput: param.value,\n        locate,\n      });\n    } else if (type === 'KeyboardPress') {\n      const param = plan.param as PlanningActionParamInputOrKeyPress;\n      flow.push({\n        aiKeyboardPress: param.value,\n        locate,\n      });\n    } else if (type === 'Scroll') {\n      const param = plan.param as ScrollParam;\n      flow.push({\n        aiScroll: null,\n        locate,\n        direction: param.direction,\n        scrollType: param.scrollType,\n        distance: param.distance,\n      });\n    } else if (type === 'Sleep') {\n      const param = plan.param as PlanningActionParamSleep;\n      flow.push({\n        sleep: param.timeMs,\n      });\n    } else if (\n      type === 'AndroidBackButton' ||\n      type === 'AndroidHomeButton' ||\n      type === 'AndroidRecentAppsButton' ||\n      type === 'AndroidLongPress' ||\n      type === 'AndroidPull'\n    ) {\n      // not implemented in yaml yet\n    } else if (\n      type === 'Error' ||\n      type === 'Assert' ||\n      type === 'AssertWithoutThrow' ||\n      type === 'Finished'\n    ) {\n      // do nothing\n    } else {\n      console.warn(\n        `Cannot convert action ${type} to yaml flow. This should be a bug of Midscene.`,\n      );\n    }\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep: sleep,\n    });\n  }\n\n  return flow;\n}\n"],"names":["AIActionType","callAiFn","msgs","AIActionTypeValue","jsonObject","callToGetJSONObject","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","vlLocateMode","adaptGeminiBbox","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rect","warned","warnGPT4oSizeLimit","size","_getModelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","sleep","flow","plan","_plan_locate","type","param"],"mappings":";;;;;;;AAoCO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;WAAZA;;AAQL,eAAeC,SACpBC,IAAY,EACZC,iBAA+B;IAE/B,MAAMC,aAAa,MAAMC,oBAAuBH,MAAMC;IAEtD,OAAO;QACL,SAASC,WAAW,OAAO;QAC3B,OAAOA,WAAW,KAAK;IACzB;AACF;AAEA,MAAME,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc;IAGd,IAAKF,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGG,UAAUH,OAAO,IAAI,EAAEC,OAAOC;IAG9C,OAAOF;AACT;AAEO,SAASI,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;QACN,YAAnB,OAAOS,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;KAC1B;IACD,OAAOa;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCJ,KAAa,EACbC,MAAc;IAEdU,OACEX,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOG,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;YAC5CQ,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;SAC7C;QAEH,MAAM,IAAIM,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAIH,IACEc,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;QAEhEc,KAAK,GAAG,CACNT,OACAS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACNR,QACAQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIS,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAGH,MAAMI,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdJ,KAAa,EACbC,MAAc;IAEd,IAAIkB,AAAmB,oBAAnBA,kBAAsCA,AAAmB,kBAAnBA,gBACxC,OAAOT,gBAAgBN,MAAMJ,OAAOC;IAGtC,IAAIkB,AAAmB,aAAnBA,gBACF,OAAOC,gBAAgBhB,MAAMJ,OAAOC;IAGtC,OAAOE,cAAcC;AACvB;AAEO,SAASgB,gBACdhB,IAAc,EACdJ,KAAa,EACbC,MAAc;IAEd,MAAMoB,OAAOZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC5C,MAAMsB,MAAMb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC5C,MAAMsB,QAAQd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC7C,MAAMwB,SAASf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC/C,OAAO;QAACoB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdrB,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdyB,UAAU,CAAC,EACXC,UAAU,CAAC;IAEX/B,kBAAkB,mBAAmBQ,MAAMJ,OAAOC,QAAQyB,SAASC;IACnE,MAAM,CAACN,MAAMC,KAAKC,OAAOC,OAAO,GAAGtB,UAAUE,MAAMJ,OAAOC;IAC1D,MAAM2B,OAAO;QACX,MAAMP,OAAOK;QACb,KAAKJ,MAAMK;QACX,OAAOJ,QAAQF;QACf,QAAQG,SAASF;IACnB;IACA1B,kBAAkB,4BAA4BgC;IAC9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBAAmBC,IAAU;QAEvCC;IADJ,IAAIH,QAAQ;IACZ,IAAI,QAAAG,CAAAA,gBAAAA,cAAa,IAAbA,KAAAA,IAAAA,cAAgB,WAAW,GAAG,QAAQ,CAAC,WAAW;QACpD,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,wFAAwF,CAAC;QAEhN,IACEtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpCtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBAAiBd,IAAU,EAAEe,UAAgB;IAC3D,MAAMC,cAAczB,AAAmB,oBAAnBA,iBAAqC,MAAM;IAC/D,MAAM0B,iBAAiB;IAEvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IACNjB,KAAK,IAAI,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,IAAI,GAAGkB;IACpClB,KAAK,KAAK,GAAGnB,KAAK,GAAG,CACnBmB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA,uBACbH,WAAW,KAAK,GAAGf,KAAK,IAAI;IAE9BA,KAAK,GAAG,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,GAAG,GAAGmB;IAClCnB,KAAK,MAAM,GAAGnB,KAAK,GAAG,CACpBmB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA,qBACdJ,WAAW,MAAM,GAAGf,KAAK,GAAG;IAE9B,OAAOA;AACT;AAEO,eAAeoB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCnB,IAAU;IAEV,MAAMoB,eAAeC,WAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtBtB;IACF;IACA,OAAOyB;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQH,MAAO;YAETI;QADf,MAAMC,OAAOF,KAAK,IAAI;QACtB,MAAM/D,SAAS,QAAAgE,CAAAA,eAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,MAAM;QAElC,IAAIC,AAAS,UAATA,MACFH,KAAK,IAAI,CAAC;YACR,OAAO9D;QACT;aACK,IAAIiE,AAAS,YAATA,MACTH,KAAK,IAAI,CAAC;YACR,SAAS9D;QACX;aACK,IAAIiE,AAAS,YAATA,MAAkB;YAC3B,MAAMC,QAAQH,KAAK,KAAK;YACxBD,KAAK,IAAI,CAAC;gBACR,SAASI,MAAM,KAAK;gBACpBlE;YACF;QACF,OAAO,IAAIiE,AAAS,oBAATA,MAA0B;YACnC,MAAMC,QAAQH,KAAK,KAAK;YACxBD,KAAK,IAAI,CAAC;gBACR,iBAAiBI,MAAM,KAAK;gBAC5BlE;YACF;QACF,OAAO,IAAIiE,AAAS,aAATA,MAAmB;YAC5B,MAAMC,QAAQH,KAAK,KAAK;YACxBD,KAAK,IAAI,CAAC;gBACR,UAAU;gBACV9D;gBACA,WAAWkE,MAAM,SAAS;gBAC1B,YAAYA,MAAM,UAAU;gBAC5B,UAAUA,MAAM,QAAQ;YAC1B;QACF,OAAO,IAAID,AAAS,YAATA,MAAkB;YAC3B,MAAMC,QAAQH,KAAK,KAAK;YACxBD,KAAK,IAAI,CAAC;gBACR,OAAOI,MAAM,MAAM;YACrB;QACF,OACW,wBAATD,QACAA,AAAS,wBAATA,QACAA,AAAS,8BAATA,QACAA,AAAS,uBAATA,QACAA,AAAS,kBAATA,QAIAA,AAAS,YAATA,QACAA,AAAS,aAATA,QACAA,AAAS,yBAATA,QACAA,AAAS,eAATA,QAIA9B,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAE8B,KAAK,gDAAgD,CAAC;IAGrF;IAEA,IAAIJ,OACFC,KAAK,IAAI,CAAC;QACR,OAAOD;IACT;IAGF,OAAOC;AACT"}

package/dist/es/ai-model/index.mjs

@@ -3,8 +3,8 @@ import { systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { describeUserPage, elementByPositionWithElementInfo } from "./prompt/util.mjs";
 import { generatePlaywrightTest, generatePlaywrightTestStream } from "./prompt/playwright-generator.mjs";
 import { generateYamlTest, generateYamlTestStream } from "./prompt/yaml-generator.mjs";
-import { AiExtractElementInfo, AiLocateElement, AiLocateSection } from "./inspect.mjs";
+import { AiAssert, AiExtractElementInfo, AiLocateElement, AiLocateSection } from "./inspect.mjs";
 import { plan } from "./llm-planning.mjs";
-import { AIActionType, actionSpaceTypePrefix, adaptBboxToRect, callAiFn } from "./common.mjs";
+import { AIActionType, adaptBboxToRect, callAiFn } from "./common.mjs";
 import { resizeImageForUiTars, vlmPlanning } from "./ui-tars-planning.mjs";
-export { AIActionType, AiExtractElementInfo, AiLocateElement, AiLocateSection, actionSpaceTypePrefix, adaptBboxToRect, call as callAi, callAiFn, callAiFnWithStringResponse, callToGetJSONObject, describeUserPage, elementByPositionWithElementInfo, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, plan, resizeImageForUiTars, systemPromptToLocateElement, vlmPlanning };
+export { AIActionType, AiAssert, AiExtractElementInfo, AiLocateElement, AiLocateSection, adaptBboxToRect, call as callAi, callAiFn, callAiFnWithStringResponse, callToGetJSONObject, describeUserPage, elementByPositionWithElementInfo, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, plan, resizeImageForUiTars, systemPromptToLocateElement, vlmPlanning };

package/dist/es/ai-model/inspect.mjs

@@ -1,8 +1,9 @@
-import { MIDSCENE_USE_QWEN_VL, getAIConfigInBoolean, vlLocateMode } from "@midscene/shared/env";
+import { MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, getAIConfigInBoolean, vlLocateMode } from "@midscene/shared/env";
 import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { AIActionType, adaptBboxToRect, callAiFn, expandSearchArea, markupImageForLLM, mergeRects } from "./common.mjs";
+import { systemPromptToAssert } from "./prompt/assertion.mjs";
 import { extractDataQueryPrompt, systemPromptToExtract } from "./prompt/extraction.mjs";
 import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
@@ -263,6 +264,54 @@ async function AiExtractElementInfo(options) {
         usage: result.usage
     };
 }
-export { AiExtractElementInfo, AiLocateElement, AiLocateSection };
+async function AiAssert(options) {
+    const { assertion, context } = options;
+    assert(assertion, 'assertion should not be empty');
+    const { screenshotBase64 } = context;
+    const systemPrompt = systemPromptToAssert({
+        isUITars: getAIConfigInBoolean(MIDSCENE_USE_VLM_UI_TARS)
+    });
+    const assertionText = extraTextFromUserPrompt(assertion);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: screenshotBase64,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: `
+Here is the assertion. Please tell whether it is truthy according to the screenshot.
+=====================================
+${assertionText}
+=====================================
+  `
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof assertion) {
+        const addOns = await promptsToChatParam({
+            images: assertion.images,
+            convertHttpImage2Base64: assertion.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const { content: assertResult, usage } = await callAiFn(msgs, AIActionType.ASSERT);
+    return {
+        content: assertResult,
+        usage
+    };
+}
+export { AiAssert, AiExtractElementInfo, AiLocateElement, AiLocateSection };
 
 //# sourceMappingURL=inspect.mjs.map

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocatorResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  BaseElement,\n  ElementById,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  TMultimodalPrompt,\n  TUserPrompt,\n  UIContext,\n} from '@/types';\nimport {\n  MIDSCENE_USE_QWEN_VL,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  callAiFn,\n  expandSearchArea,\n  markupImageForLLM,\n  mergeRects,\n} from './common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  describeUserPage,\n  distance,\n  distanceThreshold,\n  elementByPositionWithElementInfo,\n} from './prompt/util';\nimport { callToGetJSONObject } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement<\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  context: UIContext<ElementType>;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAI?: typeof callAiFn<AIElementResponse | [number, number]>;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n}): Promise<{\n  parseResult: AIElementLocatorResponse;\n  rect?: Rect;\n  rawResponse: string;\n  elementById: ElementById;\n  usage?: AIUsageInfo;\n  isOrderSensitive?: boolean;\n}> {\n  const { context, targetElementDescription, callAI } = options;\n  const { screenshotBase64 } = context;\n  const { description, elementById, insertElementByPosition } =\n    await describeUserPage(context);\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: description,\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(vlLocateMode());\n\n  let imagePayload = screenshotBase64;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n  } else if (vlLocateMode() === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlLocateMode()) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const callAIFn =\n    callAI || callToGetJSONObject<AIElementResponse | [number, number]>;\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: AIElementLocatorResponse['elements'] =\n    'elements' in res.content ? res.content.elements : [];\n  let errors: AIElementLocatorResponse['errors'] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        options.searchConfig?.rect?.width || context.size.width,\n        options.searchConfig?.rect?.height || context.size.height,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n      );\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n      let element = elementByPositionWithElementInfo(context.tree, rectCenter);\n\n      const distanceToCenter = element\n        ? distance({ x: element.center[0], y: element.center[1] }, rectCenter)\n        : 0;\n\n      if (!element || distanceToCenter > distanceThreshold) {\n        element = insertElementByPosition(rectCenter);\n      }\n\n      if (element) {\n        matchedElements = [element];\n        errors = [];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements,\n      errors,\n    },\n    rawResponse,\n    elementById,\n    usage: res.usage,\n    isOrderSensitive:\n      typeof res.content === 'object' &&\n      res.content !== null &&\n      'isOrderSensitive' in res.content\n        ? (res.content as any).isOrderSensitive\n        : undefined,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext<BaseElement>;\n  sectionDescription: TUserPrompt;\n  callAI?: typeof callAiFn<AISectionLocatorResponse>;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlLocateMode());\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAiFn<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(bbox, context.size.width, context.size.height);\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    imageBase64 = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL),\n    );\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<\n  T,\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext<ElementType>;\n  extractOption?: InsightExtractOption;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt } = options;\n  const systemPrompt = systemPromptToExtract();\n\n  const { screenshotBase64 } = context;\n  const { description, elementById } = await describeUserPage(context, {\n    truncateTextLength: 200,\n    filterNonTextContent: false,\n    visibleOnly: false,\n    domIncluded: extractOption?.domIncluded,\n  });\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    description,\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (options.extractOption?.returnThought) {\n    msgs.push({\n      role: 'user',\n      content: 'Please provide reasons.',\n    });\n  }\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAiFn<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n  );\n  return {\n    parseResult: result.content,\n    elementById,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAI","screenshotBase64","description","elementById","insertElementByPosition","describeUserPage","assert","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","vlLocateMode","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","addOns","callAIFn","callToGetJSONObject","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect","_options_searchConfig_rect1","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","elementByPositionWithElementInfo","distanceToCenter","distance","distanceThreshold","e","msg","Error","undefined","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAiFn","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","cropByRect","getAIConfigInBoolean","MIDSCENE_USE_QWEN_VL","AiExtractElementInfo","_options_extractOption","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent"],"mappings":";;;;;;;;;;AAgEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAEpBC,OAMD;IAQC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,MAAM,EAAE,GAAGH;IACtD,MAAM,EAAEI,gBAAgB,EAAE,GAAGH;IAC7B,MAAM,EAAEI,WAAW,EAAEC,WAAW,EAAEC,uBAAuB,EAAE,GACzD,MAAMC,iBAAiBP;IAEzBQ,OACEP,0BACA;IAGF,MAAMQ,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiBN;QACjB,0BAA0Bf,wBAAwBY;IACpD;IACA,MAAMU,eAAeC,4BAA4BC;IAEjD,IAAIC,eAAeX;IAEnB,IAAIJ,QAAQ,YAAY,EAAE;QACxBS,OACET,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFS,OACET,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFe,eAAef,QAAQ,YAAY,CAAC,WAAW;IACjD,OAAO,IAAIc,AAAmB,cAAnBA,gBACTC,eAAe,MAAMC,4BAA4BD;SAC5C,IAAI,CAACD,gBACVC,eAAe,MAAME,kBACnBb,kBACAH,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhB,MAAMN,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKG;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAML;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOR,0BAAuC;QAChD,MAAMgB,SAAS,MAAM1B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAMC,WACJhB,UAAUiB;IAEZ,MAAMC,MAAM,MAAMF,SAASxB,MAAM2B,aAAa,eAAe;IAE7D,MAAMC,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBACF,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IACvD,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAG1DQ,4BAAAA,uBACAC,6BAAAA,wBACAC,6BAAAA,wBACAC,6BAAAA;YALFP,UAAUQ,gBACRZ,IAAI,OAAO,CAAC,IAAI,EAChBQ,AAAAA,SAAAA,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,6BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,2BAA4B,KAAK,AAAD,KAAK5B,QAAQ,IAAI,CAAC,KAAK,EACvD6B,AAAAA,SAAAA,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,MAAM,AAAD,KAAK7B,QAAQ,IAAI,CAAC,MAAM,UACzD8B,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG;YAEjC7C,aAAa,WAAWsC;YAExB,MAAMS,aAAa;gBACjB,GAAGT,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YACA,IAAIU,UAAUC,iCAAiCnC,QAAQ,IAAI,EAAEiC;YAE7D,MAAMG,mBAAmBF,UACrBG,SAAS;gBAAE,GAAGH,QAAQ,MAAM,CAAC,EAAE;gBAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;YAAC,GAAGD,cACzD;YAEJ,IAAI,CAACC,WAAWE,mBAAmBE,mBACjCJ,UAAU5B,wBAAwB2B;YAGpC,IAAIC,SAAS;gBACXT,kBAAkB;oBAACS;iBAAQ;gBAC3BR,SAAS,EAAE;YACb;QACF;IACF,EAAE,OAAOa,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACb,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEc,IAAI,CAAC,CAAC;aAFtBd,SAAS;YAACc;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMhB;QACN,aAAa;YACX,UAAUC;YACVC;QACF;QACAJ;QACAjB;QACA,OAAOe,IAAI,KAAK;QAChB,kBACE,AAAuB,YAAvB,OAAOA,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCsB;IACR;AACF;AAEO,eAAeC,gBAAgB5C,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAE4C,kBAAkB,EAAE,GAAG7C;IACxC,MAAM,EAAEI,gBAAgB,EAAE,GAAGH;IAE7B,MAAMW,eAAekC,4BAA4BhC;IACjD,MAAMiC,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoB1D,wBAAwBuD;IAC9C;IACA,MAAMlD,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM2C;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAM3B,SAAS,MAAM1B,mBAAmB;YACtC,QAAQqD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACAlD,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAM+B,SAAS,MAAMC,SACnBvD,MACA2B,aAAa,YAAY;IAG3B,IAAI6B;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAapB,gBACjBmB,aACAnD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM;QAErBZ,aAAa,0BAA0BgE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9D5D,aAAa,wBAAwBiE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAS5B,MAAM,OAAO,CAAC4B,OAC/B,GAAG,CAAC,CAACA,OACGvB,gBAAgBuB,MAAMvD,QAAQ,IAAI,CAAC,KAAK,EAAEA,QAAQ,IAAI,CAAC,MAAM;QAExEZ,aAAa,qBAAqBkE;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DlE,aAAa,iBAAiBoE;QAG9BN,cAAcQ,iBAAiBF,YAAYxD,QAAQ,IAAI;QACvDZ,aAAa,2BAA2B8D;IAC1C;IAEA,IAAIS,cAAcxD;IAClB,IAAI+C,aACFS,cAAc,MAAMC,WAClBzD,kBACA+C,aACAW,qBAAqBC;IAIzB,OAAO;QACL,MAAMZ;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAazB,KAAK,SAAS,CAACyB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAee,qBAGpBhE,OAKD;QA0CKiE;IAzCJ,MAAM,EAAEC,SAAS,EAAEjE,OAAO,EAAEkE,aAAa,EAAE1E,gBAAgB,EAAE,GAAGO;IAChE,MAAMY,eAAewD;IAErB,MAAM,EAAEhE,gBAAgB,EAAE,GAAGH;IAC7B,MAAM,EAAEI,WAAW,EAAEC,WAAW,EAAE,GAAG,MAAME,iBAAiBP,SAAS;QACnE,oBAAoB;QACpB,sBAAsB;QACtB,aAAa;QACb,aAAakE,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,WAAW;IACzC;IAEA,MAAME,wBAAwB,MAAMC,uBAClCjE,aACA6D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKnE;YACL,QAAQ;QACV;IACF;IAGFmE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM1E,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS2D;QACX;KACD;IAED,IAAI,QAAAN,CAAAA,yBAAAA,QAAQ,aAAa,AAAD,IAApBA,KAAAA,IAAAA,uBAAuB,aAAa,EACtCtE,KAAK,IAAI,CAAC;QACR,MAAM;QACN,SAAS;IACX;IAGF,IAAIF,kBAAkB;QACpB,MAAMyB,SAAS,MAAM1B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAM+B,SAAS,MAAMC,SACnBvD,MACA2B,aAAa,YAAY;IAE3B,OAAO;QACL,aAAa2B,OAAO,OAAO;QAC3B3C;QACA,OAAO2C,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIAssertionResponse,\n  AIDataExtractionResponse,\n  AIElementLocatorResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  BaseElement,\n  ElementById,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  TMultimodalPrompt,\n  TUserPrompt,\n  UIContext,\n} from '@/types';\nimport {\n  MIDSCENE_USE_QWEN_VL,\n  MIDSCENE_USE_VLM_UI_TARS,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  callAiFn,\n  expandSearchArea,\n  markupImageForLLM,\n  mergeRects,\n} from './common';\nimport { systemPromptToAssert } from './prompt/assertion';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  describeUserPage,\n  distance,\n  distanceThreshold,\n  elementByPositionWithElementInfo,\n} from './prompt/util';\nimport { callToGetJSONObject } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement<\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  context: UIContext<ElementType>;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAI?: typeof callAiFn<AIElementResponse | [number, number]>;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n}): Promise<{\n  parseResult: AIElementLocatorResponse;\n  rect?: Rect;\n  rawResponse: string;\n  elementById: ElementById;\n  usage?: AIUsageInfo;\n  isOrderSensitive?: boolean;\n}> {\n  const { context, targetElementDescription, callAI } = options;\n  const { screenshotBase64 } = context;\n  const { description, elementById, insertElementByPosition } =\n    await describeUserPage(context);\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: description,\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(vlLocateMode());\n\n  let imagePayload = screenshotBase64;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n  } else if (vlLocateMode() === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlLocateMode()) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const callAIFn =\n    callAI || callToGetJSONObject<AIElementResponse | [number, number]>;\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: AIElementLocatorResponse['elements'] =\n    'elements' in res.content ? res.content.elements : [];\n  let errors: AIElementLocatorResponse['errors'] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        options.searchConfig?.rect?.width || context.size.width,\n        options.searchConfig?.rect?.height || context.size.height,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n      );\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n      let element = elementByPositionWithElementInfo(context.tree, rectCenter);\n\n      const distanceToCenter = element\n        ? distance({ x: element.center[0], y: element.center[1] }, rectCenter)\n        : 0;\n\n      if (!element || distanceToCenter > distanceThreshold) {\n        element = insertElementByPosition(rectCenter);\n      }\n\n      if (element) {\n        matchedElements = [element];\n        errors = [];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements,\n      errors,\n    },\n    rawResponse,\n    elementById,\n    usage: res.usage,\n    isOrderSensitive:\n      typeof res.content === 'object' &&\n      res.content !== null &&\n      'isOrderSensitive' in res.content\n        ? (res.content as any).isOrderSensitive\n        : undefined,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext<BaseElement>;\n  sectionDescription: TUserPrompt;\n  callAI?: typeof callAiFn<AISectionLocatorResponse>;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlLocateMode());\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAiFn<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(bbox, context.size.width, context.size.height);\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    imageBase64 = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL),\n    );\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<\n  T,\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext<ElementType>;\n  extractOption?: InsightExtractOption;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt } = options;\n  const systemPrompt = systemPromptToExtract();\n\n  const { screenshotBase64 } = context;\n  const { description, elementById } = await describeUserPage(context, {\n    truncateTextLength: 200,\n    filterNonTextContent: false,\n    visibleOnly: false,\n    domIncluded: extractOption?.domIncluded,\n  });\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    description,\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (options.extractOption?.returnThought) {\n    msgs.push({\n      role: 'user',\n      content: 'Please provide reasons.',\n    });\n  }\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAiFn<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n  );\n  return {\n    parseResult: result.content,\n    elementById,\n    usage: result.usage,\n  };\n}\n\nexport async function AiAssert<\n  ElementType extends BaseElement = BaseElement,\n>(options: { assertion: TUserPrompt; context: UIContext<ElementType> }) {\n  const { assertion, context } = options;\n\n  assert(assertion, 'assertion should not be empty');\n\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToAssert({\n    isUITars: getAIConfigInBoolean(MIDSCENE_USE_VLM_UI_TARS),\n  });\n\n  const assertionText = extraTextFromUserPrompt(assertion);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: `\nHere is the assertion. Please tell whether it is truthy according to the screenshot.\n=====================================\n${assertionText}\n=====================================\n  `,\n        },\n      ],\n    },\n  ];\n\n  if (typeof assertion !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: assertion.images,\n      convertHttpImage2Base64: assertion.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const { content: assertResult, usage } = await callAiFn<AIAssertionResponse>(\n    msgs,\n    AIActionType.ASSERT,\n  );\n  return {\n    content: assertResult,\n    usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAI","screenshotBase64","description","elementById","insertElementByPosition","describeUserPage","assert","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","vlLocateMode","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","addOns","callAIFn","callToGetJSONObject","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect","_options_searchConfig_rect1","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","elementByPositionWithElementInfo","distanceToCenter","distance","distanceThreshold","e","msg","Error","undefined","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAiFn","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","cropByRect","getAIConfigInBoolean","MIDSCENE_USE_QWEN_VL","AiExtractElementInfo","_options_extractOption","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiAssert","assertion","systemPromptToAssert","MIDSCENE_USE_VLM_UI_TARS","assertionText","assertResult","usage"],"mappings":";;;;;;;;;;;AAmEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAEpBC,OAMD;IAQC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,MAAM,EAAE,GAAGH;IACtD,MAAM,EAAEI,gBAAgB,EAAE,GAAGH;IAC7B,MAAM,EAAEI,WAAW,EAAEC,WAAW,EAAEC,uBAAuB,EAAE,GACzD,MAAMC,iBAAiBP;IAEzBQ,OACEP,0BACA;IAGF,MAAMQ,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiBN;QACjB,0BAA0Bf,wBAAwBY;IACpD;IACA,MAAMU,eAAeC,4BAA4BC;IAEjD,IAAIC,eAAeX;IAEnB,IAAIJ,QAAQ,YAAY,EAAE;QACxBS,OACET,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFS,OACET,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFe,eAAef,QAAQ,YAAY,CAAC,WAAW;IACjD,OAAO,IAAIc,AAAmB,cAAnBA,gBACTC,eAAe,MAAMC,4BAA4BD;SAC5C,IAAI,CAACD,gBACVC,eAAe,MAAME,kBACnBb,kBACAH,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhB,MAAMN,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKG;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAML;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOR,0BAAuC;QAChD,MAAMgB,SAAS,MAAM1B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAMC,WACJhB,UAAUiB;IAEZ,MAAMC,MAAM,MAAMF,SAASxB,MAAM2B,aAAa,eAAe;IAE7D,MAAMC,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBACF,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IACvD,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAG1DQ,4BAAAA,uBACAC,6BAAAA,wBACAC,6BAAAA,wBACAC,6BAAAA;YALFP,UAAUQ,gBACRZ,IAAI,OAAO,CAAC,IAAI,EAChBQ,AAAAA,SAAAA,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,6BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,2BAA4B,KAAK,AAAD,KAAK5B,QAAQ,IAAI,CAAC,KAAK,EACvD6B,AAAAA,SAAAA,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,MAAM,AAAD,KAAK7B,QAAQ,IAAI,CAAC,MAAM,UACzD8B,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG;YAEjC7C,aAAa,WAAWsC;YAExB,MAAMS,aAAa;gBACjB,GAAGT,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YACA,IAAIU,UAAUC,iCAAiCnC,QAAQ,IAAI,EAAEiC;YAE7D,MAAMG,mBAAmBF,UACrBG,SAAS;gBAAE,GAAGH,QAAQ,MAAM,CAAC,EAAE;gBAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;YAAC,GAAGD,cACzD;YAEJ,IAAI,CAACC,WAAWE,mBAAmBE,mBACjCJ,UAAU5B,wBAAwB2B;YAGpC,IAAIC,SAAS;gBACXT,kBAAkB;oBAACS;iBAAQ;gBAC3BR,SAAS,EAAE;YACb;QACF;IACF,EAAE,OAAOa,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACb,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEc,IAAI,CAAC,CAAC;aAFtBd,SAAS;YAACc;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMhB;QACN,aAAa;YACX,UAAUC;YACVC;QACF;QACAJ;QACAjB;QACA,OAAOe,IAAI,KAAK;QAChB,kBACE,AAAuB,YAAvB,OAAOA,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCsB;IACR;AACF;AAEO,eAAeC,gBAAgB5C,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAE4C,kBAAkB,EAAE,GAAG7C;IACxC,MAAM,EAAEI,gBAAgB,EAAE,GAAGH;IAE7B,MAAMW,eAAekC,4BAA4BhC;IACjD,MAAMiC,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoB1D,wBAAwBuD;IAC9C;IACA,MAAMlD,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM2C;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAM3B,SAAS,MAAM1B,mBAAmB;YACtC,QAAQqD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACAlD,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAM+B,SAAS,MAAMC,SACnBvD,MACA2B,aAAa,YAAY;IAG3B,IAAI6B;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAapB,gBACjBmB,aACAnD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM;QAErBZ,aAAa,0BAA0BgE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9D5D,aAAa,wBAAwBiE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAS5B,MAAM,OAAO,CAAC4B,OAC/B,GAAG,CAAC,CAACA,OACGvB,gBAAgBuB,MAAMvD,QAAQ,IAAI,CAAC,KAAK,EAAEA,QAAQ,IAAI,CAAC,MAAM;QAExEZ,aAAa,qBAAqBkE;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DlE,aAAa,iBAAiBoE;QAG9BN,cAAcQ,iBAAiBF,YAAYxD,QAAQ,IAAI;QACvDZ,aAAa,2BAA2B8D;IAC1C;IAEA,IAAIS,cAAcxD;IAClB,IAAI+C,aACFS,cAAc,MAAMC,WAClBzD,kBACA+C,aACAW,qBAAqBC;IAIzB,OAAO;QACL,MAAMZ;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAazB,KAAK,SAAS,CAACyB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAee,qBAGpBhE,OAKD;QA0CKiE;IAzCJ,MAAM,EAAEC,SAAS,EAAEjE,OAAO,EAAEkE,aAAa,EAAE1E,gBAAgB,EAAE,GAAGO;IAChE,MAAMY,eAAewD;IAErB,MAAM,EAAEhE,gBAAgB,EAAE,GAAGH;IAC7B,MAAM,EAAEI,WAAW,EAAEC,WAAW,EAAE,GAAG,MAAME,iBAAiBP,SAAS;QACnE,oBAAoB;QACpB,sBAAsB;QACtB,aAAa;QACb,aAAakE,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,WAAW;IACzC;IAEA,MAAME,wBAAwB,MAAMC,uBAClCjE,aACA6D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKnE;YACL,QAAQ;QACV;IACF;IAGFmE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM1E,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS2D;QACX;KACD;IAED,IAAI,QAAAN,CAAAA,yBAAAA,QAAQ,aAAa,AAAD,IAApBA,KAAAA,IAAAA,uBAAuB,aAAa,EACtCtE,KAAK,IAAI,CAAC;QACR,MAAM;QACN,SAAS;IACX;IAGF,IAAIF,kBAAkB;QACpB,MAAMyB,SAAS,MAAM1B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAM+B,SAAS,MAAMC,SACnBvD,MACA2B,aAAa,YAAY;IAE3B,OAAO;QACL,aAAa2B,OAAO,OAAO;QAC3B3C;QACA,OAAO2C,OAAO,KAAK;IACrB;AACF;AAEO,eAAeuB,SAEpBxE,OAAoE;IACpE,MAAM,EAAEyE,SAAS,EAAExE,OAAO,EAAE,GAAGD;IAE/BS,OAAOgE,WAAW;IAElB,MAAM,EAAErE,gBAAgB,EAAE,GAAGH;IAE7B,MAAMW,eAAe8D,qBAAqB;QACxC,UAAUZ,qBAAqBa;IACjC;IAEA,MAAMC,gBAAgBtF,wBAAwBmF;IAE9C,MAAM9E,OAAe;QACnB;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM,CAAC;;;AAGjB,EAAEwE,cAAc;;EAEd,CAAC;gBACK;aACD;QACH;KACD;IAED,IAAI,AAAqB,YAArB,OAAOH,WAAwB;QACjC,MAAMvD,SAAS,MAAM1B,mBAAmB;YACtC,QAAQiF,UAAU,MAAM;YACxB,yBAAyBA,UAAU,uBAAuB;QAC5D;QACA9E,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAM,EAAE,SAAS2D,YAAY,EAAEC,KAAK,EAAE,GAAG,MAAM5B,SAC7CvD,MACA2B,aAAa,MAAM;IAErB,OAAO;QACL,SAASuD;QACTC;IACF;AACF"}

package/dist/es/ai-model/llm-planning.mjs

@@ -56,7 +56,7 @@ async function plan(userInstruction, opts) {
         actions,
         rawResponse,
         usage,
-        yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace, planFromAI.sleep)
+        yamlFlow: buildYamlFlowFromPlans(actions, planFromAI.sleep)
     };
     assert(planFromAI, "can't get plans from AI");
     if (vlLocateMode()) {

package/dist/es/ai-model/llm-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.mjs","sources":["webpack://@midscene/core/./src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n  DeviceAction,\n  PageType,\n  PlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport { vlLocateMode } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { assert } from '@midscene/shared/utils';\nimport {\n  AIActionType,\n  type AIArgs,\n  buildYamlFlowFromPlans,\n  callAiFn,\n  fillBboxParam,\n  markupImageForLLM,\n  warnGPT4oSizeLimit,\n} from './common';\nimport {\n  automationUserPrompt,\n  generateTaskBackgroundContext,\n  systemPromptToTaskPlanning,\n} from './prompt/llm-planning';\nimport { describeUserPage } from './prompt/util';\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    pageType: PageType;\n    actionSpace: DeviceAction[];\n    callAI?: typeof callAiFn<PlanningAIResponse>;\n    log?: string;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { callAI, context } = opts || {};\n  const { screenshotBase64, size } = context;\n  const { description: pageDescription, elementById } =\n    await describeUserPage(context);\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode: vlLocateMode(),\n  });\n  const taskBackgroundContextText = generateTaskBackgroundContext(\n    userInstruction,\n    opts.log,\n    opts.actionContext,\n  );\n  const userInstructionPrompt = await automationUserPrompt(\n    vlLocateMode(),\n  ).format({\n    pageDescription,\n    taskBackgroundContext: taskBackgroundContextText,\n  });\n\n  let imagePayload = screenshotBase64;\n  if (vlLocateMode() === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlLocateMode()) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  warnGPT4oSizeLimit(size);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  const call = callAI || callAiFn;\n  const { content, usage } = await call(msgs, AIActionType.PLAN);\n  const rawResponse = JSON.stringify(content, undefined, 2);\n  const planFromAI = content;\n\n  const actions =\n    (planFromAI.action?.type ? [planFromAI.action] : planFromAI.actions) || [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  if (vlLocateMode()) {\n    actions.forEach((action) => {\n      if (action.locate) {\n        try {\n          action.locate = fillBboxParam(action.locate, size.width, size.height);\n        } catch (e) {\n          throw new Error(\n            `Failed to fill locate param: ${planFromAI.error} (${\n              e instanceof Error ? e.message : 'unknown error'\n            })`,\n            {\n              cause: e,\n            },\n          );\n        }\n      }\n    });\n    // in Qwen-VL, error means error. In GPT-4o, error may mean more actions are needed.\n    assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);\n  } else {\n    actions.forEach((action) => {\n      if (action.locate?.id) {\n        // The model may return indexId, need to perform a query correction to avoid exceptions\n        const element = elementById(action.locate.id);\n        if (element) {\n          action.locate.id = element.id;\n        }\n      }\n    });\n  }\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  return returnValue;\n}\n"],"names":["plan","userInstruction","opts","_planFromAI_action","callAI","context","screenshotBase64","size","pageDescription","elementById","describeUserPage","systemPrompt","systemPromptToTaskPlanning","vlLocateMode","taskBackgroundContextText","generateTaskBackgroundContext","userInstructionPrompt","automationUserPrompt","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","warnGPT4oSizeLimit","msgs","call","callAiFn","content","usage","AIActionType","rawResponse","JSON","undefined","planFromAI","actions","returnValue","buildYamlFlowFromPlans","assert","action","fillBboxParam","e","Error","_action_locate","element","console"],"mappings":";;;;;;AAyBO,eAAeA,KACpBC,eAAuB,EACvBC,IAOC;QA8DEC;IA5DH,MAAM,EAAEC,MAAM,EAAEC,OAAO,EAAE,GAAGH,QAAQ,CAAC;IACrC,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGF;IACnC,MAAM,EAAE,aAAaG,eAAe,EAAEC,WAAW,EAAE,GACjD,MAAMC,iBAAiBL;IAEzB,MAAMM,eAAe,MAAMC,2BAA2B;QACpD,aAAaV,KAAK,WAAW;QAC7B,QAAQW;IACV;IACA,MAAMC,4BAA4BC,8BAChCd,iBACAC,KAAK,GAAG,EACRA,KAAK,aAAa;IAEpB,MAAMc,wBAAwB,MAAMC,qBAClCJ,gBACA,MAAM,CAAC;QACPL;QACA,uBAAuBM;IACzB;IAEA,IAAII,eAAeZ;IACnB,IAAIO,AAAmB,cAAnBA,gBACFK,eAAe,MAAMC,4BAA4BD;SAC5C,IAAI,CAACL,gBACVK,eAAe,MAAME,kBACnBd,kBACAD,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhBgB,mBAAmBd;IAEnB,MAAMe,OAAe;QACnB;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKO;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMF;gBACR;aACD;QACH;KACD;IAED,MAAMO,OAAOnB,UAAUoB;IACvB,MAAM,EAAEC,OAAO,EAAEC,KAAK,EAAE,GAAG,MAAMH,KAAKD,MAAMK,aAAa,IAAI;IAC7D,MAAMC,cAAcC,KAAK,SAAS,CAACJ,SAASK,QAAW;IACvD,MAAMC,aAAaN;IAEnB,MAAMO,UACH7B,AAAAA,CAAAA,SAAAA,CAAAA,qBAAAA,WAAW,MAAM,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,IAAI;QAAC4B,WAAW,MAAM;KAAC,GAAGA,WAAW,OAAM,KAAM,EAAE;IAC5E,MAAME,cAAkC;QACtC,GAAGF,UAAU;QACbC;QACAJ;QACAF;QACA,UAAUQ,uBACRF,SACA9B,KAAK,WAAW,EAChB6B,WAAW,KAAK;IAEpB;IAEAI,OAAOJ,YAAY;IAEnB,IAAIlB,gBAAgB;QAClBmB,QAAQ,OAAO,CAAC,CAACI;YACf,IAAIA,OAAO,MAAM,EACf,IAAI;gBACFA,OAAO,MAAM,GAAGC,cAAcD,OAAO,MAAM,EAAE7B,KAAK,KAAK,EAAEA,KAAK,MAAM;YACtE,EAAE,OAAO+B,GAAG;gBACV,MAAM,IAAIC,MACR,CAAC,6BAA6B,EAAER,WAAW,KAAK,CAAC,EAAE,EACjDO,aAAaC,QAAQD,EAAE,OAAO,GAAG,gBAClC,CAAC,CAAC,EACH;oBACE,OAAOA;gBACT;YAEJ;QAEJ;QAEAH,OAAO,CAACJ,WAAW,KAAK,EAAE,CAAC,wBAAwB,EAAEA,WAAW,KAAK,EAAE;IACzE,OACEC,QAAQ,OAAO,CAAC,CAACI;YACXI;QAAJ,IAAI,QAAAA,CAAAA,iBAAAA,OAAO,MAAM,AAAD,IAAZA,KAAAA,IAAAA,eAAe,EAAE,EAAE;YAErB,MAAMC,UAAUhC,YAAY2B,OAAO,MAAM,CAAC,EAAE;YAC5C,IAAIK,SACFL,OAAO,MAAM,CAAC,EAAE,GAAGK,QAAQ,EAAE;QAEjC;IACF;IAGF,IACET,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBS,QAAQ,IAAI,CACV,8EACAzC;IAIJ,OAAOgC;AACT"}
+{"version":3,"file":"ai-model/llm-planning.mjs","sources":["webpack://@midscene/core/./src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n  DeviceAction,\n  PageType,\n  PlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport { vlLocateMode } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { assert } from '@midscene/shared/utils';\nimport {\n  AIActionType,\n  type AIArgs,\n  buildYamlFlowFromPlans,\n  callAiFn,\n  fillBboxParam,\n  markupImageForLLM,\n  warnGPT4oSizeLimit,\n} from './common';\nimport {\n  automationUserPrompt,\n  generateTaskBackgroundContext,\n  systemPromptToTaskPlanning,\n} from './prompt/llm-planning';\nimport { describeUserPage } from './prompt/util';\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    pageType: PageType;\n    actionSpace: DeviceAction[];\n    callAI?: typeof callAiFn<PlanningAIResponse>;\n    log?: string;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { callAI, context } = opts || {};\n  const { screenshotBase64, size } = context;\n  const { description: pageDescription, elementById } =\n    await describeUserPage(context);\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode: vlLocateMode(),\n  });\n  const taskBackgroundContextText = generateTaskBackgroundContext(\n    userInstruction,\n    opts.log,\n    opts.actionContext,\n  );\n  const userInstructionPrompt = await automationUserPrompt(\n    vlLocateMode(),\n  ).format({\n    pageDescription,\n    taskBackgroundContext: taskBackgroundContextText,\n  });\n\n  let imagePayload = screenshotBase64;\n  if (vlLocateMode() === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlLocateMode()) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  warnGPT4oSizeLimit(size);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  const call = callAI || callAiFn;\n  const { content, usage } = await call(msgs, AIActionType.PLAN);\n  const rawResponse = JSON.stringify(content, undefined, 2);\n  const planFromAI = content;\n\n  const actions =\n    (planFromAI.action?.type ? [planFromAI.action] : planFromAI.actions) || [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(actions, planFromAI.sleep),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  if (vlLocateMode()) {\n    actions.forEach((action) => {\n      if (action.locate) {\n        try {\n          action.locate = fillBboxParam(action.locate, size.width, size.height);\n        } catch (e) {\n          throw new Error(\n            `Failed to fill locate param: ${planFromAI.error} (${\n              e instanceof Error ? e.message : 'unknown error'\n            })`,\n            {\n              cause: e,\n            },\n          );\n        }\n      }\n    });\n    // in Qwen-VL, error means error. In GPT-4o, error may mean more actions are needed.\n    assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);\n  } else {\n    actions.forEach((action) => {\n      if (action.locate?.id) {\n        // The model may return indexId, need to perform a query correction to avoid exceptions\n        const element = elementById(action.locate.id);\n        if (element) {\n          action.locate.id = element.id;\n        }\n      }\n    });\n  }\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  return returnValue;\n}\n"],"names":["plan","userInstruction","opts","_planFromAI_action","callAI","context","screenshotBase64","size","pageDescription","elementById","describeUserPage","systemPrompt","systemPromptToTaskPlanning","vlLocateMode","taskBackgroundContextText","generateTaskBackgroundContext","userInstructionPrompt","automationUserPrompt","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","warnGPT4oSizeLimit","msgs","call","callAiFn","content","usage","AIActionType","rawResponse","JSON","undefined","planFromAI","actions","returnValue","buildYamlFlowFromPlans","assert","action","fillBboxParam","e","Error","_action_locate","element","console"],"mappings":";;;;;;AAyBO,eAAeA,KACpBC,eAAuB,EACvBC,IAOC;QA8DEC;IA5DH,MAAM,EAAEC,MAAM,EAAEC,OAAO,EAAE,GAAGH,QAAQ,CAAC;IACrC,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGF;IACnC,MAAM,EAAE,aAAaG,eAAe,EAAEC,WAAW,EAAE,GACjD,MAAMC,iBAAiBL;IAEzB,MAAMM,eAAe,MAAMC,2BAA2B;QACpD,aAAaV,KAAK,WAAW;QAC7B,QAAQW;IACV;IACA,MAAMC,4BAA4BC,8BAChCd,iBACAC,KAAK,GAAG,EACRA,KAAK,aAAa;IAEpB,MAAMc,wBAAwB,MAAMC,qBAClCJ,gBACA,MAAM,CAAC;QACPL;QACA,uBAAuBM;IACzB;IAEA,IAAII,eAAeZ;IACnB,IAAIO,AAAmB,cAAnBA,gBACFK,eAAe,MAAMC,4BAA4BD;SAC5C,IAAI,CAACL,gBACVK,eAAe,MAAME,kBACnBd,kBACAD,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhBgB,mBAAmBd;IAEnB,MAAMe,OAAe;QACnB;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKO;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMF;gBACR;aACD;QACH;KACD;IAED,MAAMO,OAAOnB,UAAUoB;IACvB,MAAM,EAAEC,OAAO,EAAEC,KAAK,EAAE,GAAG,MAAMH,KAAKD,MAAMK,aAAa,IAAI;IAC7D,MAAMC,cAAcC,KAAK,SAAS,CAACJ,SAASK,QAAW;IACvD,MAAMC,aAAaN;IAEnB,MAAMO,UACH7B,AAAAA,CAAAA,SAAAA,CAAAA,qBAAAA,WAAW,MAAM,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,IAAI;QAAC4B,WAAW,MAAM;KAAC,GAAGA,WAAW,OAAM,KAAM,EAAE;IAC5E,MAAME,cAAkC;QACtC,GAAGF,UAAU;QACbC;QACAJ;QACAF;QACA,UAAUQ,uBAAuBF,SAASD,WAAW,KAAK;IAC5D;IAEAI,OAAOJ,YAAY;IAEnB,IAAIlB,gBAAgB;QAClBmB,QAAQ,OAAO,CAAC,CAACI;YACf,IAAIA,OAAO,MAAM,EACf,IAAI;gBACFA,OAAO,MAAM,GAAGC,cAAcD,OAAO,MAAM,EAAE7B,KAAK,KAAK,EAAEA,KAAK,MAAM;YACtE,EAAE,OAAO+B,GAAG;gBACV,MAAM,IAAIC,MACR,CAAC,6BAA6B,EAAER,WAAW,KAAK,CAAC,EAAE,EACjDO,aAAaC,QAAQD,EAAE,OAAO,GAAG,gBAClC,CAAC,CAAC,EACH;oBACE,OAAOA;gBACT;YAEJ;QAEJ;QAEAH,OAAO,CAACJ,WAAW,KAAK,EAAE,CAAC,wBAAwB,EAAEA,WAAW,KAAK,EAAE;IACzE,OACEC,QAAQ,OAAO,CAAC,CAACI;YACXI;QAAJ,IAAI,QAAAA,CAAAA,iBAAAA,OAAO,MAAM,AAAD,IAAZA,KAAAA,IAAAA,eAAe,EAAE,EAAE;YAErB,MAAMC,UAAUhC,YAAY2B,OAAO,MAAM,CAAC,EAAE;YAC5C,IAAIK,SACFL,OAAO,MAAM,CAAC,EAAE,GAAGK,QAAQ,EAAE;QAEjC;IACF;IAGF,IACET,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBS,QAAQ,IAAI,CACV,8EACAzC;IAIJ,OAAOgC;AACT"}

package/dist/es/ai-model/prompt/assertion.mjs

@@ -1,3 +1,27 @@
+import { getPreferredLanguage } from "@midscene/shared/env";
+const defaultAssertionPrompt = 'You are a senior testing engineer. User will give an assertion and a screenshot of a page. By carefully viewing the screenshot, please tell whether the assertion is truthy.';
+const defaultAssertionResponseJsonFormat = `Return in the following JSON format:
+{
+  pass: boolean, // whether the assertion is truthy
+  thought: string | null, // string, if the result is falsy, give the reason why it is falsy. Otherwise, put null.
+}`;
+const getUiTarsAssertionResponseJsonFormat = ()=>`## Output Json String Format
+\`\`\`
+"{
+  "pass": <<is a boolean value from the enum [true, false], true means the assertion is truthy>>, 
+  "thought": "<<is a string, give the reason why the assertion is falsy or truthy. Otherwise.>>"
+}"
+\`\`\`
+
+## Rules **MUST** follow
+- Make sure to return **only** the JSON, with **no additional** text or explanations.
+- Use ${getPreferredLanguage()} in \`thought\` part.
+- You **MUST** strictly follow up the **Output Json String Format**.`;
+function systemPromptToAssert(model) {
+    return `${defaultAssertionPrompt}
+
+${model.isUITars ? getUiTarsAssertionResponseJsonFormat() : defaultAssertionResponseJsonFormat}`;
+}
 const assertSchema = {
     type: 'json_schema',
     json_schema: {
@@ -26,6 +50,6 @@ const assertSchema = {
         }
     }
 };
-export { assertSchema };
+export { assertSchema, systemPromptToAssert };
 
 //# sourceMappingURL=assertion.mjs.map

package/dist/es/ai-model/prompt/assertion.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/assertion.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/assertion.ts"],"sourcesContent":["import type { ResponseFormatJSONSchema } from 'openai/resources/index';\n\nexport const assertSchema: ResponseFormatJSONSchema = {\n  type: 'json_schema',\n  json_schema: {\n    name: 'assert',\n    strict: true,\n    schema: {\n      type: 'object',\n      properties: {\n        pass: {\n          type: 'boolean',\n          description: 'Whether the assertion passed or failed',\n        },\n        thought: {\n          type: ['string', 'null'],\n          description: 'The thought process behind the assertion',\n        },\n      },\n      required: ['pass', 'thought'],\n      additionalProperties: false,\n    },\n  },\n};\n"],"names":["assertSchema"],"mappings":"AAEO,MAAMA,eAAyC;IACpD,MAAM;IACN,aAAa;QACX,MAAM;QACN,QAAQ;QACR,QAAQ;YACN,MAAM;YACN,YAAY;gBACV,MAAM;oBACJ,MAAM;oBACN,aAAa;gBACf;gBACA,SAAS;oBACP,MAAM;wBAAC;wBAAU;qBAAO;oBACxB,aAAa;gBACf;YACF;YACA,UAAU;gBAAC;gBAAQ;aAAU;YAC7B,sBAAsB;QACxB;IACF;AACF"}
+{"version":3,"file":"ai-model/prompt/assertion.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/assertion.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\n\nconst defaultAssertionPrompt =\n  'You are a senior testing engineer. User will give an assertion and a screenshot of a page. By carefully viewing the screenshot, please tell whether the assertion is truthy.';\n\nconst defaultAssertionResponseJsonFormat = `Return in the following JSON format:\n{\n  pass: boolean, // whether the assertion is truthy\n  thought: string | null, // string, if the result is falsy, give the reason why it is falsy. Otherwise, put null.\n}`;\n\nconst getUiTarsAssertionResponseJsonFormat = () => `## Output Json String Format\n\\`\\`\\`\n\"{\n  \"pass\": <<is a boolean value from the enum [true, false], true means the assertion is truthy>>, \n  \"thought\": \"<<is a string, give the reason why the assertion is falsy or truthy. Otherwise.>>\"\n}\"\n\\`\\`\\`\n\n## Rules **MUST** follow\n- Make sure to return **only** the JSON, with **no additional** text or explanations.\n- Use ${getPreferredLanguage()} in \\`thought\\` part.\n- You **MUST** strictly follow up the **Output Json String Format**.`;\n\nexport function systemPromptToAssert(model: { isUITars: boolean }) {\n  return `${defaultAssertionPrompt}\n\n${model.isUITars ? getUiTarsAssertionResponseJsonFormat() : defaultAssertionResponseJsonFormat}`;\n}\n\nexport const assertSchema: ResponseFormatJSONSchema = {\n  type: 'json_schema',\n  json_schema: {\n    name: 'assert',\n    strict: true,\n    schema: {\n      type: 'object',\n      properties: {\n        pass: {\n          type: 'boolean',\n          description: 'Whether the assertion passed or failed',\n        },\n        thought: {\n          type: ['string', 'null'],\n          description: 'The thought process behind the assertion',\n        },\n      },\n      required: ['pass', 'thought'],\n      additionalProperties: false,\n    },\n  },\n};\n"],"names":["defaultAssertionPrompt","defaultAssertionResponseJsonFormat","getUiTarsAssertionResponseJsonFormat","getPreferredLanguage","systemPromptToAssert","model","assertSchema"],"mappings":";AAGA,MAAMA,yBACJ;AAEF,MAAMC,qCAAqC,CAAC;;;;CAI3C,CAAC;AAEF,MAAMC,uCAAuC,IAAM,CAAC;;;;;;;;;;MAU9C,EAAEC,uBAAuB;oEACqC,CAAC;AAE9D,SAASC,qBAAqBC,KAA4B;IAC/D,OAAO,GAAGL,uBAAuB;;AAEnC,EAAEK,MAAM,QAAQ,GAAGH,yCAAyCD,oCAAoC;AAChG;AAEO,MAAMK,eAAyC;IACpD,MAAM;IACN,aAAa;QACX,MAAM;QACN,QAAQ;QACR,QAAQ;YACN,MAAM;YACN,YAAY;gBACV,MAAM;oBACJ,MAAM;oBACN,aAAa;gBACf;gBACA,SAAS;oBACP,MAAM;wBAAC;wBAAU;qBAAO;oBACxB,aAAa;gBACf;YACF;YACA,UAAU;gBAAC;gBAAQ;aAAU;YAC7B,sBAAsB;QACxB;IACF;AACF"}

package/dist/es/index.mjs

@@ -1,9 +1,9 @@
 import { Executor } from "./ai-model/action-executor.mjs";
 import insight from "./insight/index.mjs";
 import { getVersion } from "./utils.mjs";
-import { AiLocateElement, describeUserPage, plan } from "./ai-model/index.mjs";
+import { AiAssert, AiLocateElement, describeUserPage, plan } from "./ai-model/index.mjs";
 import { MIDSCENE_MODEL_NAME, getAIConfig } from "@midscene/shared/env";
 const src = insight;
-export { AiLocateElement, Executor, insight as Insight, MIDSCENE_MODEL_NAME, src as default, describeUserPage, getAIConfig, getVersion, plan };
+export { AiAssert, AiLocateElement, Executor, insight as Insight, MIDSCENE_MODEL_NAME, src as default, describeUserPage, getAIConfig, getVersion, plan };
 
 //# sourceMappingURL=index.mjs.map

package/dist/es/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","sources":["webpack://@midscene/core/./src/index.ts"],"sourcesContent":["import { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n} from './ai-model/index';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';\n\nexport type * from './types';\nexport default Insight;\nexport { Executor, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlFlowItemAIRightClick,\n  MidsceneYamlConfigResult,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n"],"names":["Insight"],"mappings":";;;;;AAaA,YAAeA"}
+{"version":3,"file":"index.mjs","sources":["webpack://@midscene/core/./src/index.ts"],"sourcesContent":["import { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  AiAssert,\n} from './ai-model/index';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';\n\nexport type * from './types';\nexport default Insight;\nexport { Executor, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlFlowItemAIRightClick,\n  MidsceneYamlConfigResult,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n"],"names":["Insight"],"mappings":";;;;;AAcA,YAAeA"}

package/dist/es/insight/index.mjs

@@ -1,6 +1,6 @@
 import { AIActionType, callAiFn, expandSearchArea } from "../ai-model/common.mjs";
 import { AiExtractElementInfo, AiLocateElement, callToGetJSONObject } from "../ai-model/index.mjs";
-import { AiLocateSection } from "../ai-model/inspect.mjs";
+import { AiAssert, AiLocateSection } from "../ai-model/inspect.mjs";
 import { elementDescriberInstruction } from "../ai-model/prompt/describe.mjs";
 import { MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_USE_QWEN_VL, getAIConfigInBoolean, vlLocateMode } from "@midscene/shared/env";
 import { compositeElementInfoImg, cropByRect } from "@midscene/shared/img";
@@ -156,6 +156,41 @@ class Insight {
             usage
         };
     }
+    async assert(assertion) {
+        const dumpSubscriber = this.onceDumpUpdatedFn;
+        this.onceDumpUpdatedFn = void 0;
+        const context = await this.contextRetrieverFn('assert');
+        const startTime = Date.now();
+        const assertResult = await AiAssert({
+            assertion,
+            context
+        });
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse: JSON.stringify(assertResult.content)
+        };
+        const { thought, pass } = assertResult.content;
+        const dumpData = {
+            type: 'assert',
+            userQuery: {
+                assertion
+            },
+            matchedElement: [],
+            data: null,
+            taskInfo,
+            assertionPass: pass,
+            assertionThought: thought,
+            error: pass ? void 0 : thought
+        };
+        emitInsightDump(dumpData, dumpSubscriber);
+        return {
+            pass,
+            thought,
+            usage: assertResult.usage
+        };
+    }
     async describe(target, opt) {
         assert(target, 'target is required for insight.describe');
         const context = await this.contextRetrieverFn('describe');