@midscene/core 0.26.7-beta-20250821033353.0 → 0.26.7-beta-20250821074936.0

package/dist/es/ai-model/common.mjs

@@ -14,7 +14,6 @@ var common_AIActionType = /*#__PURE__*/ function(AIActionType) {
     AIActionType[AIActionType["DESCRIBE_ELEMENT"] = 4] = "DESCRIBE_ELEMENT";
     return AIActionType;
 }({});
-const actionSpaceTypePrefix = 'action_space_';
 async function callAiFn(msgs, AIActionTypeValue, modelPreferences) {
     const jsonObject = await callToGetJSONObject(msgs, AIActionTypeValue, modelPreferences);
     return {
@@ -174,18 +173,17 @@ async function markupImageForLLM(screenshotBase64, tree, size) {
 function buildYamlFlowFromPlans(plans, actionSpace, sleep) {
     const flow = [];
     for (const plan of plans){
-        var _plan_locate;
         const verb = plan.type;
         const action = actionSpace.find((action)=>action.name === verb);
         if (!action) {
             console.warn(`Cannot convert action ${verb} to yaml flow. Will ignore it.`);
             continue;
         }
-        const locate = null == (_plan_locate = plan.locate) ? void 0 : _plan_locate.prompt;
-        const flowKey = action.interfaceAlias || `${actionSpaceTypePrefix}${verb}`;
+        const flowKey = action.interfaceAlias || verb;
+        const flowParam = action.paramSchema ? dumpActionParam(plan.param || {}, action.paramSchema) : {};
         const flowItem = {
-            [flowKey]: locate || '',
-            ...plan.param || {}
+            [flowKey]: '',
+            ...flowParam
         };
         flow.push(flowItem);
     }
@@ -206,35 +204,103 @@ const SizeSchema = z.object({
 const RectSchema = PointSchema.and(SizeSchema).and(z.object({
     zoom: z.number().optional()
 }));
-const MidsceneLocation = z.object({
-    midscene_location_field_flag: z.literal(true),
-    prompt: z.string(),
+const TMultimodalPromptSchema = z.object({
+    images: z.array(z.object({
+        name: z.string(),
+        url: z.string()
+    })).optional(),
+    convertHttpImage2Base64: z.boolean().optional()
+});
+const TUserPromptSchema = z.union([
+    z.string(),
+    z.object({
+        prompt: z.string()
+    }).and(TMultimodalPromptSchema.partial())
+]);
+const locateFieldFlagName = 'midscene_location_field_flag';
+const MidsceneLocationResult = z.object({
+    [locateFieldFlagName]: z.literal(true),
+    prompt: TUserPromptSchema,
+    deepThink: z.boolean().optional(),
+    cacheable: z.boolean().optional(),
+    xpath: z.boolean().optional(),
     center: z.tuple([
         z.number(),
         z.number()
     ]),
     rect: RectSchema
 }).passthrough();
+const getMidsceneLocationSchema = ()=>MidsceneLocationResult;
 const ifMidsceneLocatorField = (field)=>{
     var _actualField__def, _actualField__def1;
     let actualField = field;
     if ((null == (_actualField__def = actualField._def) ? void 0 : _actualField__def.typeName) === 'ZodOptional') actualField = actualField._def.innerType;
     if ((null == (_actualField__def1 = actualField._def) ? void 0 : _actualField__def1.typeName) === 'ZodObject') {
         const shape = actualField._def.shape();
-        return 'midscene_location_field_flag' in shape;
+        return locateFieldFlagName in shape;
     }
     return false;
 };
-const findAllMidsceneLocatorField = (zodType)=>{
+const dumpMidsceneLocatorField = (field)=>{
+    assert(ifMidsceneLocatorField(field), 'field is not a midscene locator field');
+    if ('string' == typeof field) return field;
+    if (field && 'object' == typeof field && field.prompt) {
+        if ('string' == typeof field.prompt) return field.prompt;
+        if ('object' == typeof field.prompt && field.prompt.prompt) return field.prompt.prompt;
+    }
+    return String(field);
+};
+const findAllMidsceneLocatorField = (zodType, requiredOnly)=>{
     var _zodObject__def;
     if (!zodType) return [];
     const zodObject = zodType;
     if ((null == (_zodObject__def = zodObject._def) ? void 0 : _zodObject__def.typeName) === 'ZodObject' && zodObject.shape) {
         const keys = Object.keys(zodObject.shape);
-        return keys.filter((key)=>ifMidsceneLocatorField(zodObject.shape[key]));
+        return keys.filter((key)=>{
+            const field = zodObject.shape[key];
+            if (!ifMidsceneLocatorField(field)) return false;
+            if (requiredOnly) {
+                var _field__def;
+                return (null == (_field__def = field._def) ? void 0 : _field__def.typeName) !== 'ZodOptional';
+            }
+            return true;
+        });
     }
     return [];
 };
-export { common_AIActionType as AIActionType, MidsceneLocation, PointSchema, RectSchema, SizeSchema, actionSpaceTypePrefix, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptQwenBbox, buildYamlFlowFromPlans, callAiFn, expandSearchArea, fillBboxParam, findAllMidsceneLocatorField, ifMidsceneLocatorField, markupImageForLLM, mergeRects, warnGPT4oSizeLimit };
+const dumpActionParam = (jsonObject, zodSchema)=>{
+    const locatorFields = findAllMidsceneLocatorField(zodSchema);
+    const result = {
+        ...jsonObject
+    };
+    for (const fieldName of locatorFields){
+        const fieldValue = result[fieldName];
+        if (fieldValue) {
+            if ('string' == typeof fieldValue) result[fieldName] = fieldValue;
+            else if ('object' == typeof fieldValue) {
+                if (fieldValue.prompt) {
+                    if ('string' == typeof fieldValue.prompt) result[fieldName] = fieldValue.prompt;
+                    else if ('object' == typeof fieldValue.prompt && fieldValue.prompt.prompt) result[fieldName] = fieldValue.prompt.prompt;
+                }
+            }
+        }
+    }
+    return result;
+};
+const loadActionParam = (jsonObject, zodSchema)=>{
+    const locatorFields = findAllMidsceneLocatorField(zodSchema);
+    const result = {
+        ...jsonObject
+    };
+    for (const fieldName of locatorFields){
+        const fieldValue = result[fieldName];
+        if (fieldValue && 'string' == typeof fieldValue) result[fieldName] = {
+            [locateFieldFlagName]: true,
+            prompt: fieldValue
+        };
+    }
+    return result;
+};
+export { common_AIActionType as AIActionType, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptQwenBbox, buildYamlFlowFromPlans, callAiFn, dumpActionParam, dumpMidsceneLocatorField, expandSearchArea, fillBboxParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, ifMidsceneLocatorField, loadActionParam, markupImageForLLM, mergeRects, warnGPT4oSizeLimit };
 
 //# sourceMappingURL=common.mjs.map

package/dist/es/ai-model/common.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport { callToGetJSONObject } from './service-caller/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  type IModelPreferences,\n  getModelName,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n}\n\nexport const actionSpaceTypePrefix = 'action_space_';\n\nexport async function callAiFn<T>(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelPreferences: IModelPreferences,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const jsonObject = await callToGetJSONObject<T>(\n    msgs,\n    AIActionTypeValue,\n    modelPreferences,\n  );\n\n  return {\n    content: jsonObject.content,\n    usage: jsonObject.usage,\n  };\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(locate.bbox, width, height, modelPreferences);\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n): [number, number, number, number] {\n  if (\n    vlLocateMode(modelPreferences) === 'doubao-vision' ||\n    vlLocateMode(modelPreferences) === 'vlm-ui-tars'\n  ) {\n    return adaptDoubaoBbox(bbox, width, height);\n  }\n\n  if (vlLocateMode(modelPreferences) === 'gemini') {\n    return adaptGeminiBbox(bbox, width, height);\n  }\n\n  return adaptQwenBbox(bbox);\n}\n\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n  offsetX = 0,\n  offsetY = 0,\n): Rect {\n  debugInspectUtils('adaptBboxToRect', bbox, width, height, offsetX, offsetY);\n  const [left, top, right, bottom] = adaptBbox(\n    bbox,\n    width,\n    height,\n    modelPreferences,\n  );\n  const rect = {\n    left: left + offsetX,\n    top: top + offsetY,\n    width: right - left,\n    height: bottom - top,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(\n  size: Size,\n  modelPreferences: IModelPreferences,\n) {\n  if (warned) return;\n  if (getModelName(modelPreferences)?.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your page to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  modelPreferences: IModelPreferences,\n) {\n  const minEdgeSize =\n    vlLocateMode(modelPreferences) === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n  rect.left = Math.max(0, rect.left - paddingSizeHorizontal);\n  rect.width = Math.min(\n    rect.width + paddingSizeHorizontal * 2,\n    screenSize.width - rect.left,\n  );\n  rect.top = Math.max(0, rect.top - paddingSizeVertical);\n  rect.height = Math.min(\n    rect.height + paddingSizeVertical * 2,\n    screenSize.height - rect.top,\n  );\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const locate = plan.locate?.prompt;\n    const flowKey = action.interfaceAlias || `${actionSpaceTypePrefix}${verb}`;\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: locate || '',\n      ...(plan.param || {}),\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\nexport const MidsceneLocation = z\n  .object({\n    midscene_location_field_flag: z.literal(true),\n    prompt: z.string(),\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\nexport type MidsceneLocationType = z.infer<typeof MidsceneLocation>;\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject with midscene_location_field_flag\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n    return 'midscene_location_field_flag' in shape;\n  }\n\n  return false;\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => ifMidsceneLocatorField(zodObject.shape[key]));\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n"],"names":["AIActionType","actionSpaceTypePrefix","callAiFn","msgs","AIActionTypeValue","modelPreferences","jsonObject","callToGetJSONObject","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","vlLocateMode","adaptGeminiBbox","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rect","warned","warnGPT4oSizeLimit","size","_getModelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","_plan_locate","verb","action","flowKey","flowItem","PointSchema","z","SizeSchema","RectSchema","MidsceneLocation","ifMidsceneLocatorField","field","_actualField__def","_actualField__def1","actualField","shape","findAllMidsceneLocatorField","zodType","_zodObject__def","zodObject","keys","Object","key"],"mappings":";;;;;;;;AAmCO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;WAAZA;;AAQL,MAAMC,wBAAwB;AAE9B,eAAeC,SACpBC,IAAY,EACZC,iBAA+B,EAC/BC,gBAAmC;IAEnC,MAAMC,aAAa,MAAMC,oBACvBJ,MACAC,mBACAC;IAGF,OAAO;QACL,SAASC,WAAW,OAAO;QAC3B,OAAOA,WAAW,KAAK;IACzB;AACF;AAEA,MAAME,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdT,gBAAmC;IAGnC,IAAKO,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGG,UAAUH,OAAO,IAAI,EAAEC,OAAOC,QAAQT;IAGtD,OAAOO;AACT;AAEO,SAASI,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;QACN,YAAnB,OAAOS,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;KAC1B;IACD,OAAOa;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCJ,KAAa,EACbC,MAAc;IAEdU,OACEX,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOG,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;YAC5CQ,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;SAC7C;QAEH,MAAM,IAAIM,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAIH,IACEc,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;QAEhEc,KAAK,GAAG,CACNT,OACAS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACNR,QACAQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIS,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAGH,MAAMI,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdT,gBAAmC;IAEnC,IACE2B,AAAmC,oBAAnCA,aAAa3B,qBACb2B,AAAmC,kBAAnCA,aAAa3B,mBAEb,OAAOkB,gBAAgBN,MAAMJ,OAAOC;IAGtC,IAAIkB,AAAmC,aAAnCA,aAAa3B,mBACf,OAAO4B,gBAAgBhB,MAAMJ,OAAOC;IAGtC,OAAOE,cAAcC;AACvB;AAEO,SAASgB,gBACdhB,IAAc,EACdJ,KAAa,EACbC,MAAc;IAEd,MAAMoB,OAAOZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC5C,MAAMsB,MAAMb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC5C,MAAMsB,QAAQd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC7C,MAAMwB,SAASf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC/C,OAAO;QAACoB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdrB,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdT,gBAAmC,EACnCkC,UAAU,CAAC,EACXC,UAAU,CAAC;IAEX/B,kBAAkB,mBAAmBQ,MAAMJ,OAAOC,QAAQyB,SAASC;IACnE,MAAM,CAACN,MAAMC,KAAKC,OAAOC,OAAO,GAAGtB,UACjCE,MACAJ,OACAC,QACAT;IAEF,MAAMoC,OAAO;QACX,MAAMP,OAAOK;QACb,KAAKJ,MAAMK;QACX,OAAOJ,QAAQF;QACf,QAAQG,SAASF;IACnB;IACA1B,kBAAkB,4BAA4BgC;IAC9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBACdC,IAAU,EACVvC,gBAAmC;QAG/BwC;IADJ,IAAIH,QAAQ;IACZ,IAAI,QAAAG,CAAAA,gBAAAA,aAAaxC,iBAAgB,IAA7BwC,KAAAA,IAAAA,cAAgC,WAAW,GAAG,QAAQ,CAAC,WAAW;QACpE,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,wFAAwF,CAAC;QAEhN,IACEtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpCtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdd,IAAU,EACVe,UAAgB,EAChBnD,gBAAmC;IAEnC,MAAMoD,cACJzB,AAAmC,oBAAnCA,aAAa3B,oBAAwC,MAAM;IAC7D,MAAMqD,iBAAiB;IAEvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IACNjB,KAAK,IAAI,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,IAAI,GAAGkB;IACpClB,KAAK,KAAK,GAAGnB,KAAK,GAAG,CACnBmB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA,uBACbH,WAAW,KAAK,GAAGf,KAAK,IAAI;IAE9BA,KAAK,GAAG,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,GAAG,GAAGmB;IAClCnB,KAAK,MAAM,GAAGnB,KAAK,GAAG,CACpBmB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA,qBACdJ,WAAW,MAAM,GAAGf,KAAK,GAAG;IAE9B,OAAOA;AACT;AAEO,eAAeoB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCnB,IAAU;IAEV,MAAMoB,eAAeC,WAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtBtB;IACF;IACA,OAAOyB;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;YAWTK;QAVf,MAAMC,OAAOF,KAAK,IAAI;QAEtB,MAAMG,SAASN,YAAY,IAAI,CAAC,CAACM,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXhC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAE+B,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMlE,SAAS,QAAAiE,CAAAA,eAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,MAAM;QAClC,MAAMG,UAAUD,OAAO,cAAc,IAAI,GAAG9E,wBAAwB6E,MAAM;QAE1E,MAAMG,WAAiC;YACrC,CAACD,QAAQ,EAAEpE,UAAU;YACrB,GAAIgE,KAAK,KAAK,IAAI,CAAC,CAAC;QACtB;QAEAD,KAAK,IAAI,CAACM;IACZ;IAEA,IAAIP,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMO,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;IAChB,KAAKA,EAAE,MAAM,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAGK,MAAMG,mBAAmBH,EAAAA,MACvB,CAAC;IACN,8BAA8BA,EAAE,OAAO,CAAC;IACxC,QAAQA,EAAE,MAAM;IAChB,QAAQA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAIP,MAAME,yBAAyB,CAACC;QAGjCC,mBAKAC;IANJ,IAAIC,cAAcH;IAClB,IAAIC,AAAAA,SAAAA,CAAAA,oBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,kBAAkB,QAAQ,AAAD,MAAM,eACjCE,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAID,AAAAA,SAAAA,CAAAA,qBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,mBAAkB,QAAQ,AAAD,MAAM,aAAa;QAC9C,MAAME,QAAQD,YAAY,IAAI,CAAC,KAAK;QACpC,OAAO,kCAAkCC;IAC3C;IAEA,OAAO;AACT;AAEO,MAAMC,8BAA8B,CACzCC;QAQIC;IANJ,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIC,AAAAA,SAAAA,CAAAA,kBAAAA,UAAU,IAAI,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,QAAQ,AAAD,MAAM,eAAeC,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOC,OAAO,IAAI,CAACF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACE,MAAQZ,uBAAuBS,UAAU,KAAK,CAACG,IAAI;IACzE;IAGA,OAAO,EAAE;AACX"}
+{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport { callToGetJSONObject } from './service-caller/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  type IModelPreferences,\n  getModelName,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n}\n\nexport async function callAiFn<T>(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelPreferences: IModelPreferences,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const jsonObject = await callToGetJSONObject<T>(\n    msgs,\n    AIActionTypeValue,\n    modelPreferences,\n  );\n\n  return {\n    content: jsonObject.content,\n    usage: jsonObject.usage,\n  };\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(locate.bbox, width, height, modelPreferences);\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n): [number, number, number, number] {\n  if (\n    vlLocateMode(modelPreferences) === 'doubao-vision' ||\n    vlLocateMode(modelPreferences) === 'vlm-ui-tars'\n  ) {\n    return adaptDoubaoBbox(bbox, width, height);\n  }\n\n  if (vlLocateMode(modelPreferences) === 'gemini') {\n    return adaptGeminiBbox(bbox, width, height);\n  }\n\n  return adaptQwenBbox(bbox);\n}\n\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  modelPreferences: IModelPreferences,\n  offsetX = 0,\n  offsetY = 0,\n): Rect {\n  debugInspectUtils('adaptBboxToRect', bbox, width, height, offsetX, offsetY);\n  const [left, top, right, bottom] = adaptBbox(\n    bbox,\n    width,\n    height,\n    modelPreferences,\n  );\n  const rect = {\n    left: left + offsetX,\n    top: top + offsetY,\n    width: right - left,\n    height: bottom - top,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(\n  size: Size,\n  modelPreferences: IModelPreferences,\n) {\n  if (warned) return;\n  if (getModelName(modelPreferences)?.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your page to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  modelPreferences: IModelPreferences,\n) {\n  const minEdgeSize =\n    vlLocateMode(modelPreferences) === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n  rect.left = Math.max(0, rect.left - paddingSizeHorizontal);\n  rect.width = Math.min(\n    rect.width + paddingSizeHorizontal * 2,\n    screenSize.width - rect.left,\n  );\n  rect.top = Math.max(0, rect.top - paddingSizeVertical);\n  rect.height = Math.min(\n    rect.height + paddingSizeVertical * 2,\n    screenSize.height - rect.top,\n  );\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\nconst MidsceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationResult;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodUnion (the new MidsceneLocation structure)\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n    return locateFieldFlagName in shape;\n  }\n\n  return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n"],"names":["AIActionType","callAiFn","msgs","AIActionTypeValue","modelPreferences","jsonObject","callToGetJSONObject","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","vlLocateMode","adaptGeminiBbox","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rect","warned","warnGPT4oSizeLimit","size","_getModelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationResult","getMidsceneLocationSchema","ifMidsceneLocatorField","field","_actualField__def","_actualField__def1","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","_zodObject__def","zodObject","keys","Object","key","_field__def","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam"],"mappings":";;;;;;;;AAmCO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;WAAZA;;AAQL,eAAeC,SACpBC,IAAY,EACZC,iBAA+B,EAC/BC,gBAAmC;IAEnC,MAAMC,aAAa,MAAMC,oBACvBJ,MACAC,mBACAC;IAGF,OAAO;QACL,SAASC,WAAW,OAAO;QAC3B,OAAOA,WAAW,KAAK;IACzB;AACF;AAEA,MAAME,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdT,gBAAmC;IAGnC,IAAKO,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGG,UAAUH,OAAO,IAAI,EAAEC,OAAOC,QAAQT;IAGtD,OAAOO;AACT;AAEO,SAASI,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;QACN,YAAnB,OAAOS,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGT;KAC1B;IACD,OAAOa;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCJ,KAAa,EACbC,MAAc;IAEdU,OACEX,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOG,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;YAC5CQ,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,QAAS;YAC3CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIX,SAAU;SAC7C;QAEH,MAAM,IAAIM,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAIH,IACEc,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;QAEhEc,KAAK,GAAG,CACNT,OACAS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS,QAAQL,kBAAkB;QAE/Dc,KAAK,GAAG,CACNR,QACAQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIS,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;QACpCQ,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,QAAS;QACnCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGd,SAAU;KACrC;IAGH,MAAMI,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdT,gBAAmC;IAEnC,IACE2B,AAAmC,oBAAnCA,aAAa3B,qBACb2B,AAAmC,kBAAnCA,aAAa3B,mBAEb,OAAOkB,gBAAgBN,MAAMJ,OAAOC;IAGtC,IAAIkB,AAAmC,aAAnCA,aAAa3B,mBACf,OAAO4B,gBAAgBhB,MAAMJ,OAAOC;IAGtC,OAAOE,cAAcC;AACvB;AAEO,SAASgB,gBACdhB,IAAc,EACdJ,KAAa,EACbC,MAAc;IAEd,MAAMoB,OAAOZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC5C,MAAMsB,MAAMb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC5C,MAAMsB,QAAQd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,QAAS;IAC7C,MAAMwB,SAASf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGH,SAAU;IAC/C,OAAO;QAACoB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdrB,IAAc,EACdJ,KAAa,EACbC,MAAc,EACdT,gBAAmC,EACnCkC,UAAU,CAAC,EACXC,UAAU,CAAC;IAEX/B,kBAAkB,mBAAmBQ,MAAMJ,OAAOC,QAAQyB,SAASC;IACnE,MAAM,CAACN,MAAMC,KAAKC,OAAOC,OAAO,GAAGtB,UACjCE,MACAJ,OACAC,QACAT;IAEF,MAAMoC,OAAO;QACX,MAAMP,OAAOK;QACb,KAAKJ,MAAMK;QACX,OAAOJ,QAAQF;QACf,QAAQG,SAASF;IACnB;IACA1B,kBAAkB,4BAA4BgC;IAC9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBACdC,IAAU,EACVvC,gBAAmC;QAG/BwC;IADJ,IAAIH,QAAQ;IACZ,IAAI,QAAAG,CAAAA,gBAAAA,aAAaxC,iBAAgB,IAA7BwC,KAAAA,IAAAA,cAAgC,WAAW,GAAG,QAAQ,CAAC,WAAW;QACpE,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,wFAAwF,CAAC;QAEhN,IACEtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpCtB,KAAK,GAAG,CAACsB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdd,IAAU,EACVe,UAAgB,EAChBnD,gBAAmC;IAEnC,MAAMoD,cACJzB,AAAmC,oBAAnCA,aAAa3B,oBAAwC,MAAM;IAC7D,MAAMqD,iBAAiB;IAEvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IACNjB,KAAK,IAAI,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,IAAI,GAAGkB;IACpClB,KAAK,KAAK,GAAGnB,KAAK,GAAG,CACnBmB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA,uBACbH,WAAW,KAAK,GAAGf,KAAK,IAAI;IAE9BA,KAAK,GAAG,GAAGnB,KAAK,GAAG,CAAC,GAAGmB,KAAK,GAAG,GAAGmB;IAClCnB,KAAK,MAAM,GAAGnB,KAAK,GAAG,CACpBmB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA,qBACdJ,WAAW,MAAM,GAAGf,KAAK,GAAG;IAE9B,OAAOA;AACT;AAEO,eAAeoB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCnB,IAAU;IAEV,MAAMoB,eAAeC,WAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtBtB;IACF;IACA,OAAOyB;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACX/B,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAE8B,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAME,UAAUD,OAAO,cAAc,IAAID;QACzC,MAAMG,YAAYF,OAAO,WAAW,GAChCG,gBAAgBL,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMI,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAL,KAAK,IAAI,CAACO;IACZ;IAEA,IAAIR,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMQ,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;IAChB,KAAKA,EAAE,MAAM,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAE5B,MAAMC,yBAAyBN,EAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,EAAE,OAAO,CAAC;IACjC,QAAQI;IAGR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,OAAO,GAAG,QAAQ;IAG3B,QAAQA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAGP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;QAGjCC,mBAKAC;IANJ,IAAIC,cAAcH;IAClB,IAAIC,AAAAA,SAAAA,CAAAA,oBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,kBAAkB,QAAQ,AAAD,MAAM,eACjCE,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAID,AAAAA,SAAAA,CAAAA,qBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,mBAAkB,QAAQ,AAAD,MAAM,aAAa;QAC9C,MAAME,QAAQD,YAAY,IAAI,CAAC,KAAK;QACpC,OAAOP,uBAAuBQ;IAChC;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACL;IACvCrE,OACEoE,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOM,OAAON;AAChB;AAEO,MAAMO,8BAA8B,CACzCC,SACAC;QAQIC;IANJ,IAAI,CAACF,SACH,OAAO,EAAE;IAIX,MAAMG,YAAYH;IAClB,IAAIE,AAAAA,SAAAA,CAAAA,kBAAAA,UAAU,IAAI,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,QAAQ,AAAD,MAAM,eAAeC,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOC,OAAO,IAAI,CAACF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACE;YAClB,MAAMd,QAAQW,UAAU,KAAK,CAACG,IAAI;YAClC,IAAI,CAACf,uBAAuBC,QAC1B,OAAO;YAIT,IAAIS,cAAc;oBACTM;gBAAP,OAAOA,AAAAA,SAAAA,CAAAA,cAAAA,MAAM,IAAI,AAAD,IAATA,KAAAA,IAAAA,YAAY,QAAQ,AAAD,MAAM;YAClC;YAEA,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAM3B,kBAAkB,CAC7B3E,YACAuG;IAEA,MAAMC,gBAAgBV,4BAA4BS;IAClD,MAAMxF,SAAS;QAAE,GAAGf,UAAU;IAAC;IAE/B,KAAK,MAAMyG,aAAaD,cAAe;QACrC,MAAME,aAAa3F,MAAM,CAAC0F,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACT3F,MAAM,CAAC0F,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1B3F,MAAM,CAAC0F,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxB3F,MAAM,CAAC0F,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAO3F;AACT;AAEO,MAAM4F,kBAAkB,CAC7B3G,YACAuG;IAEA,MAAMC,gBAAgBV,4BAA4BS;IAClD,MAAMxF,SAAS;QAAE,GAAGf,UAAU;IAAC;IAE/B,KAAK,MAAMyG,aAAaD,cAAe;QACrC,MAAME,aAAa3F,MAAM,CAAC0F,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvB3F,MAAM,CAAC0F,UAAU,GAAG;YAClB,CAACtB,oBAAoB,EAAE;YACvB,QAAQuB;QACV;IAEJ;IAEA,OAAO3F;AACT"}

package/dist/es/ai-model/index.mjs

@@ -5,6 +5,6 @@ import { generatePlaywrightTest, generatePlaywrightTestStream } from "./prompt/p
 import { generateYamlTest, generateYamlTestStream } from "./prompt/yaml-generator.mjs";
 import { AiExtractElementInfo, AiLocateElement, AiLocateSection } from "./inspect.mjs";
 import { plan } from "./llm-planning.mjs";
-import { AIActionType, MidsceneLocation, PointSchema, RectSchema, SizeSchema, actionSpaceTypePrefix, adaptBboxToRect, callAiFn } from "./common.mjs";
+import { AIActionType, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, callAiFn, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, loadActionParam } from "./common.mjs";
 import { resizeImageForUiTars, vlmPlanning } from "./ui-tars-planning.mjs";
-export { AIActionType, AiExtractElementInfo, AiLocateElement, AiLocateSection, MidsceneLocation, PointSchema, RectSchema, SizeSchema, actionSpaceTypePrefix, adaptBboxToRect, call as callAi, callAiFn, callAiFnWithStringResponse, callToGetJSONObject, describeUserPage, elementByPositionWithElementInfo, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, plan, resizeImageForUiTars, systemPromptToLocateElement, vlmPlanning };
+export { AIActionType, AiExtractElementInfo, AiLocateElement, AiLocateSection, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, call as callAi, callAiFn, callAiFnWithStringResponse, callToGetJSONObject, describeUserPage, dumpActionParam, elementByPositionWithElementInfo, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, loadActionParam, plan, resizeImageForUiTars, systemPromptToLocateElement, vlmPlanning };

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocatorResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  BaseElement,\n  ElementById,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  TMultimodalPrompt,\n  TUserPrompt,\n  UIContext,\n} from '@/types';\nimport {\n  type IModelPreferences,\n  getIsUseQwenVl,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  callAiFn,\n  expandSearchArea,\n  markupImageForLLM,\n  mergeRects,\n} from './common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  describeUserPage,\n  distance,\n  distanceThreshold,\n  elementByPositionWithElementInfo,\n} from './prompt/util';\nimport { callToGetJSONObject } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement<\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  context: UIContext<ElementType>;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAI?: typeof callAiFn<AIElementResponse | [number, number]>;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n}): Promise<{\n  parseResult: AIElementLocatorResponse;\n  rect?: Rect;\n  rawResponse: string;\n  elementById: ElementById;\n  usage?: AIUsageInfo;\n  isOrderSensitive?: boolean;\n}> {\n  const { context, targetElementDescription, callAI } = options;\n  const { screenshotBase64 } = context;\n\n  const modelPreferences: IModelPreferences = {\n    intent: 'grounding',\n  };\n\n  const { description, elementById, insertElementByPosition } =\n    await describeUserPage(context, modelPreferences);\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: description,\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(\n    vlLocateMode(modelPreferences),\n  );\n\n  let imagePayload = screenshotBase64;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n  } else if (vlLocateMode(modelPreferences) === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlLocateMode(modelPreferences)) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const callAIFn =\n    callAI || callToGetJSONObject<AIElementResponse | [number, number]>;\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, {\n    intent: 'grounding',\n  });\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: AIElementLocatorResponse['elements'] =\n    'elements' in res.content ? res.content.elements : [];\n  let errors: AIElementLocatorResponse['errors'] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        options.searchConfig?.rect?.width || context.size.width,\n        options.searchConfig?.rect?.height || context.size.height,\n        modelPreferences,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n      );\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n      let element = elementByPositionWithElementInfo(context.tree, rectCenter);\n\n      const distanceToCenter = element\n        ? distance({ x: element.center[0], y: element.center[1] }, rectCenter)\n        : 0;\n\n      if (!element || distanceToCenter > distanceThreshold) {\n        element = insertElementByPosition(rectCenter);\n      }\n\n      if (element) {\n        matchedElements = [element];\n        errors = [];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements,\n      errors,\n    },\n    rawResponse,\n    elementById,\n    usage: res.usage,\n    isOrderSensitive:\n      typeof res.content === 'object' &&\n      res.content !== null &&\n      'isOrderSensitive' in res.content\n        ? (res.content as any).isOrderSensitive\n        : undefined,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext<BaseElement>;\n  sectionDescription: TUserPrompt;\n  callAI?: typeof callAiFn<AISectionLocatorResponse>;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const { screenshotBase64 } = context;\n\n  const modelPreferences: IModelPreferences = {\n    intent: 'grounding',\n  };\n\n  const systemPrompt = systemPromptToLocateSection(\n    vlLocateMode(modelPreferences),\n  );\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAiFn<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    {\n      intent: 'grounding',\n    },\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      modelPreferences,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          modelPreferences,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, modelPreferences);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    imageBase64 = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      getIsUseQwenVl({\n        intent: 'grounding',\n      }),\n    );\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<\n  T,\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext<ElementType>;\n  extractOption?: InsightExtractOption;\n  modelPreferences: IModelPreferences;\n}) {\n  const {\n    dataQuery,\n    context,\n    extractOption,\n    multimodalPrompt,\n    modelPreferences,\n  } = options;\n  const systemPrompt = systemPromptToExtract();\n\n  const { screenshotBase64 } = context;\n  const { description, elementById } = await describeUserPage(\n    context,\n    modelPreferences,\n    {\n      truncateTextLength: 200,\n      filterNonTextContent: false,\n      visibleOnly: false,\n      domIncluded: extractOption?.domIncluded,\n    },\n  );\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    description,\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (options.extractOption?.returnThought) {\n    msgs.push({\n      role: 'user',\n      content: 'Please provide reasons.',\n    });\n  }\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAiFn<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelPreferences,\n  );\n  return {\n    parseResult: result.content,\n    elementById,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAI","screenshotBase64","modelPreferences","description","elementById","insertElementByPosition","describeUserPage","assert","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","vlLocateMode","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","addOns","callAIFn","callToGetJSONObject","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect","_options_searchConfig_rect1","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","elementByPositionWithElementInfo","distanceToCenter","distance","distanceThreshold","e","msg","Error","undefined","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAiFn","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","cropByRect","getIsUseQwenVl","AiExtractElementInfo","_options_extractOption","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent"],"mappings":";;;;;;;;;;AAgEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAEpBC,OAMD;IAQC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,MAAM,EAAE,GAAGH;IACtD,MAAM,EAAEI,gBAAgB,EAAE,GAAGH;IAE7B,MAAMI,mBAAsC;QAC1C,QAAQ;IACV;IAEA,MAAM,EAAEC,WAAW,EAAEC,WAAW,EAAEC,uBAAuB,EAAE,GACzD,MAAMC,iBAAiBR,SAASI;IAElCK,OACER,0BACA;IAGF,MAAMS,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiBN;QACjB,0BAA0BhB,wBAAwBY;IACpD;IACA,MAAMW,eAAeC,4BACnBC,aAAaV;IAGf,IAAIW,eAAeZ;IAEnB,IAAIJ,QAAQ,YAAY,EAAE;QACxBU,OACEV,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFU,OACEV,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFgB,eAAehB,QAAQ,YAAY,CAAC,WAAW;IACjD,OAAO,IAAIe,AAAmC,cAAnCA,aAAaV,mBACtBW,eAAe,MAAMC,4BAA4BD;SAC5C,IAAI,CAACD,aAAaV,mBACvBW,eAAe,MAAME,kBACnBd,kBACAH,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhB,MAAMN,OAAe;QACnB;YAAE,MAAM;YAAU,SAASkB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKG;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAML;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOT,0BAAuC;QAChD,MAAMiB,SAAS,MAAM3B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAIwB;IACf;IAEA,MAAMC,WACJjB,UAAUkB;IAEZ,MAAMC,MAAM,MAAMF,SAASzB,MAAM4B,aAAa,eAAe,EAAE;QAC7D,QAAQ;IACV;IAEA,MAAMC,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBACF,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IACvD,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAG1DQ,4BAAAA,uBACAC,6BAAAA,wBAEAC,6BAAAA,wBACAC,6BAAAA;YANFP,UAAUQ,gBACRZ,IAAI,OAAO,CAAC,IAAI,EAChBQ,AAAAA,SAAAA,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,6BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,2BAA4B,KAAK,AAAD,KAAK7B,QAAQ,IAAI,CAAC,KAAK,EACvD8B,AAAAA,SAAAA,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,MAAM,AAAD,KAAK9B,QAAQ,IAAI,CAAC,MAAM,EACzDI,kBAAAA,QACA2B,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG;YAEjC9C,aAAa,WAAWuC;YAExB,MAAMS,aAAa;gBACjB,GAAGT,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YACA,IAAIU,UAAUC,iCAAiCpC,QAAQ,IAAI,EAAEkC;YAE7D,MAAMG,mBAAmBF,UACrBG,SAAS;gBAAE,GAAGH,QAAQ,MAAM,CAAC,EAAE;gBAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;YAAC,GAAGD,cACzD;YAEJ,IAAI,CAACC,WAAWE,mBAAmBE,mBACjCJ,UAAU5B,wBAAwB2B;YAGpC,IAAIC,SAAS;gBACXT,kBAAkB;oBAACS;iBAAQ;gBAC3BR,SAAS,EAAE;YACb;QACF;IACF,EAAE,OAAOa,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACb,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEc,IAAI,CAAC,CAAC;aAFtBd,SAAS;YAACc;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMhB;QACN,aAAa;YACX,UAAUC;YACVC;QACF;QACAJ;QACAjB;QACA,OAAOe,IAAI,KAAK;QAChB,kBACE,AAAuB,YAAvB,OAAOA,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCsB;IACR;AACF;AAEO,eAAeC,gBAAgB7C,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAE6C,kBAAkB,EAAE,GAAG9C;IACxC,MAAM,EAAEI,gBAAgB,EAAE,GAAGH;IAE7B,MAAMI,mBAAsC;QAC1C,QAAQ;IACV;IAEA,MAAMQ,eAAekC,4BACnBhC,aAAaV;IAEf,MAAM2C,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoB3D,wBAAwBwD;IAC9C;IACA,MAAMnD,OAAe;QACnB;YAAE,MAAM;YAAU,SAASkB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKT;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM4C;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAM3B,SAAS,MAAM3B,mBAAmB;YACtC,QAAQsD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACAnD,KAAK,IAAI,IAAIwB;IACf;IAEA,MAAM+B,SAAS,MAAMC,SACnBxD,MACA4B,aAAa,YAAY,EACzB;QACE,QAAQ;IACV;IAGF,IAAI6B;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAapB,gBACjBmB,aACApD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0BiE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9D7D,aAAa,wBAAwBkE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAS5B,MAAM,OAAO,CAAC4B,OAC/B,GAAG,CAAC,CAACA,OACGvB,gBACLuB,MACAxD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqBmE;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DnE,aAAa,iBAAiBqE;QAG9BN,cAAcQ,iBAAiBF,YAAYzD,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2B+D;IAC1C;IAEA,IAAIS,cAAczD;IAClB,IAAIgD,aACFS,cAAc,MAAMC,WAClB1D,kBACAgD,aACAW,eAAe;QACb,QAAQ;IACV;IAIJ,OAAO;QACL,MAAMX;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAazB,KAAK,SAAS,CAACyB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAGpBhE,OAMD;QAoDKiE;IAnDJ,MAAM,EACJC,SAAS,EACTjE,OAAO,EACPkE,aAAa,EACb1E,gBAAgB,EAChBY,gBAAgB,EACjB,GAAGL;IACJ,MAAMa,eAAeuD;IAErB,MAAM,EAAEhE,gBAAgB,EAAE,GAAGH;IAC7B,MAAM,EAAEK,WAAW,EAAEC,WAAW,EAAE,GAAG,MAAME,iBACzCR,SACAI,kBACA;QACE,oBAAoB;QACpB,sBAAsB;QACtB,aAAa;QACb,aAAa8D,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,WAAW;IACzC;IAGF,MAAME,wBAAwB,MAAMC,uBAClChE,aACA4D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKnE;YACL,QAAQ;QACV;IACF;IAGFmE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM1E,OAAe;QACnB;YAAE,MAAM;YAAU,SAASkB;QAAa;QACxC;YACE,MAAM;YACN,SAAS0D;QACX;KACD;IAED,IAAI,QAAAN,CAAAA,yBAAAA,QAAQ,aAAa,AAAD,IAApBA,KAAAA,IAAAA,uBAAuB,aAAa,EACtCtE,KAAK,IAAI,CAAC;QACR,MAAM;QACN,SAAS;IACX;IAGF,IAAIF,kBAAkB;QACpB,MAAM0B,SAAS,MAAM3B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAIwB;IACf;IAEA,MAAM+B,SAAS,MAAMC,SACnBxD,MACA4B,aAAa,YAAY,EACzBlB;IAEF,OAAO;QACL,aAAa6C,OAAO,OAAO;QAC3B3C;QACA,OAAO2C,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocatorResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  BaseElement,\n  ElementById,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  UIContext,\n} from '@/types';\nimport {\n  type IModelPreferences,\n  getIsUseQwenVl,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from './common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  callAiFn,\n  expandSearchArea,\n  markupImageForLLM,\n  mergeRects,\n} from './common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  describeUserPage,\n  distance,\n  distanceThreshold,\n  elementByPositionWithElementInfo,\n} from './prompt/util';\nimport { callToGetJSONObject } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement<\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  context: UIContext<ElementType>;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAI?: typeof callAiFn<AIElementResponse | [number, number]>;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n}): Promise<{\n  parseResult: AIElementLocatorResponse;\n  rect?: Rect;\n  rawResponse: string;\n  elementById: ElementById;\n  usage?: AIUsageInfo;\n  isOrderSensitive?: boolean;\n}> {\n  const { context, targetElementDescription, callAI } = options;\n  const { screenshotBase64 } = context;\n\n  const modelPreferences: IModelPreferences = {\n    intent: 'grounding',\n  };\n\n  const { description, elementById, insertElementByPosition } =\n    await describeUserPage(context, modelPreferences);\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: description,\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(\n    vlLocateMode(modelPreferences),\n  );\n\n  let imagePayload = screenshotBase64;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n  } else if (vlLocateMode(modelPreferences) === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlLocateMode(modelPreferences)) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const callAIFn =\n    callAI || callToGetJSONObject<AIElementResponse | [number, number]>;\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, {\n    intent: 'grounding',\n  });\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: AIElementLocatorResponse['elements'] =\n    'elements' in res.content ? res.content.elements : [];\n  let errors: AIElementLocatorResponse['errors'] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        options.searchConfig?.rect?.width || context.size.width,\n        options.searchConfig?.rect?.height || context.size.height,\n        modelPreferences,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n      );\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n      let element = elementByPositionWithElementInfo(context.tree, rectCenter);\n\n      const distanceToCenter = element\n        ? distance({ x: element.center[0], y: element.center[1] }, rectCenter)\n        : 0;\n\n      if (!element || distanceToCenter > distanceThreshold) {\n        element = insertElementByPosition(rectCenter);\n      }\n\n      if (element) {\n        matchedElements = [element];\n        errors = [];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements,\n      errors,\n    },\n    rawResponse,\n    elementById,\n    usage: res.usage,\n    isOrderSensitive:\n      typeof res.content === 'object' &&\n      res.content !== null &&\n      'isOrderSensitive' in res.content\n        ? (res.content as any).isOrderSensitive\n        : undefined,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext<BaseElement>;\n  sectionDescription: TUserPrompt;\n  callAI?: typeof callAiFn<AISectionLocatorResponse>;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const { screenshotBase64 } = context;\n\n  const modelPreferences: IModelPreferences = {\n    intent: 'grounding',\n  };\n\n  const systemPrompt = systemPromptToLocateSection(\n    vlLocateMode(modelPreferences),\n  );\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAiFn<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    {\n      intent: 'grounding',\n    },\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      modelPreferences,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          modelPreferences,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, modelPreferences);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    imageBase64 = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      getIsUseQwenVl({\n        intent: 'grounding',\n      }),\n    );\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<\n  T,\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext<ElementType>;\n  extractOption?: InsightExtractOption;\n  modelPreferences: IModelPreferences;\n}) {\n  const {\n    dataQuery,\n    context,\n    extractOption,\n    multimodalPrompt,\n    modelPreferences,\n  } = options;\n  const systemPrompt = systemPromptToExtract();\n\n  const { screenshotBase64 } = context;\n  const { description, elementById } = await describeUserPage(\n    context,\n    modelPreferences,\n    {\n      truncateTextLength: 200,\n      filterNonTextContent: false,\n      visibleOnly: false,\n      domIncluded: extractOption?.domIncluded,\n    },\n  );\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    description,\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (options.extractOption?.returnThought) {\n    msgs.push({\n      role: 'user',\n      content: 'Please provide reasons.',\n    });\n  }\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAiFn<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelPreferences,\n  );\n  return {\n    parseResult: result.content,\n    elementById,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAI","screenshotBase64","modelPreferences","description","elementById","insertElementByPosition","describeUserPage","assert","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","vlLocateMode","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","addOns","callAIFn","callToGetJSONObject","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect","_options_searchConfig_rect1","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","elementByPositionWithElementInfo","distanceToCenter","distance","distanceThreshold","e","msg","Error","undefined","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAiFn","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","cropByRect","getIsUseQwenVl","AiExtractElementInfo","_options_extractOption","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent"],"mappings":";;;;;;;;;;AA+DA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAEpBC,OAMD;IAQC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,MAAM,EAAE,GAAGH;IACtD,MAAM,EAAEI,gBAAgB,EAAE,GAAGH;IAE7B,MAAMI,mBAAsC;QAC1C,QAAQ;IACV;IAEA,MAAM,EAAEC,WAAW,EAAEC,WAAW,EAAEC,uBAAuB,EAAE,GACzD,MAAMC,iBAAiBR,SAASI;IAElCK,OACER,0BACA;IAGF,MAAMS,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiBN;QACjB,0BAA0BhB,wBAAwBY;IACpD;IACA,MAAMW,eAAeC,4BACnBC,aAAaV;IAGf,IAAIW,eAAeZ;IAEnB,IAAIJ,QAAQ,YAAY,EAAE;QACxBU,OACEV,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFU,OACEV,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFgB,eAAehB,QAAQ,YAAY,CAAC,WAAW;IACjD,OAAO,IAAIe,AAAmC,cAAnCA,aAAaV,mBACtBW,eAAe,MAAMC,4BAA4BD;SAC5C,IAAI,CAACD,aAAaV,mBACvBW,eAAe,MAAME,kBACnBd,kBACAH,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhB,MAAMN,OAAe;QACnB;YAAE,MAAM;YAAU,SAASkB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKG;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAML;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOT,0BAAuC;QAChD,MAAMiB,SAAS,MAAM3B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAIwB;IACf;IAEA,MAAMC,WACJjB,UAAUkB;IAEZ,MAAMC,MAAM,MAAMF,SAASzB,MAAM4B,aAAa,eAAe,EAAE;QAC7D,QAAQ;IACV;IAEA,MAAMC,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBACF,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IACvD,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAG1DQ,4BAAAA,uBACAC,6BAAAA,wBAEAC,6BAAAA,wBACAC,6BAAAA;YANFP,UAAUQ,gBACRZ,IAAI,OAAO,CAAC,IAAI,EAChBQ,AAAAA,SAAAA,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,6BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,2BAA4B,KAAK,AAAD,KAAK7B,QAAQ,IAAI,CAAC,KAAK,EACvD8B,AAAAA,SAAAA,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,MAAM,AAAD,KAAK9B,QAAQ,IAAI,CAAC,MAAM,EACzDI,kBAAAA,QACA2B,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG;YAEjC9C,aAAa,WAAWuC;YAExB,MAAMS,aAAa;gBACjB,GAAGT,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YACA,IAAIU,UAAUC,iCAAiCpC,QAAQ,IAAI,EAAEkC;YAE7D,MAAMG,mBAAmBF,UACrBG,SAAS;gBAAE,GAAGH,QAAQ,MAAM,CAAC,EAAE;gBAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;YAAC,GAAGD,cACzD;YAEJ,IAAI,CAACC,WAAWE,mBAAmBE,mBACjCJ,UAAU5B,wBAAwB2B;YAGpC,IAAIC,SAAS;gBACXT,kBAAkB;oBAACS;iBAAQ;gBAC3BR,SAAS,EAAE;YACb;QACF;IACF,EAAE,OAAOa,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACb,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEc,IAAI,CAAC,CAAC;aAFtBd,SAAS;YAACc;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMhB;QACN,aAAa;YACX,UAAUC;YACVC;QACF;QACAJ;QACAjB;QACA,OAAOe,IAAI,KAAK;QAChB,kBACE,AAAuB,YAAvB,OAAOA,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCsB;IACR;AACF;AAEO,eAAeC,gBAAgB7C,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAE6C,kBAAkB,EAAE,GAAG9C;IACxC,MAAM,EAAEI,gBAAgB,EAAE,GAAGH;IAE7B,MAAMI,mBAAsC;QAC1C,QAAQ;IACV;IAEA,MAAMQ,eAAekC,4BACnBhC,aAAaV;IAEf,MAAM2C,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoB3D,wBAAwBwD;IAC9C;IACA,MAAMnD,OAAe;QACnB;YAAE,MAAM;YAAU,SAASkB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKT;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM4C;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAM3B,SAAS,MAAM3B,mBAAmB;YACtC,QAAQsD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACAnD,KAAK,IAAI,IAAIwB;IACf;IAEA,MAAM+B,SAAS,MAAMC,SACnBxD,MACA4B,aAAa,YAAY,EACzB;QACE,QAAQ;IACV;IAGF,IAAI6B;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAapB,gBACjBmB,aACApD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0BiE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9D7D,aAAa,wBAAwBkE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAS5B,MAAM,OAAO,CAAC4B,OAC/B,GAAG,CAAC,CAACA,OACGvB,gBACLuB,MACAxD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqBmE;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DnE,aAAa,iBAAiBqE;QAG9BN,cAAcQ,iBAAiBF,YAAYzD,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2B+D;IAC1C;IAEA,IAAIS,cAAczD;IAClB,IAAIgD,aACFS,cAAc,MAAMC,WAClB1D,kBACAgD,aACAW,eAAe;QACb,QAAQ;IACV;IAIJ,OAAO;QACL,MAAMX;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAazB,KAAK,SAAS,CAACyB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAGpBhE,OAMD;QAoDKiE;IAnDJ,MAAM,EACJC,SAAS,EACTjE,OAAO,EACPkE,aAAa,EACb1E,gBAAgB,EAChBY,gBAAgB,EACjB,GAAGL;IACJ,MAAMa,eAAeuD;IAErB,MAAM,EAAEhE,gBAAgB,EAAE,GAAGH;IAC7B,MAAM,EAAEK,WAAW,EAAEC,WAAW,EAAE,GAAG,MAAME,iBACzCR,SACAI,kBACA;QACE,oBAAoB;QACpB,sBAAsB;QACtB,aAAa;QACb,aAAa8D,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,WAAW;IACzC;IAGF,MAAME,wBAAwB,MAAMC,uBAClChE,aACA4D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKnE;YACL,QAAQ;QACV;IACF;IAGFmE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM1E,OAAe;QACnB;YAAE,MAAM;YAAU,SAASkB;QAAa;QACxC;YACE,MAAM;YACN,SAAS0D;QACX;KACD;IAED,IAAI,QAAAN,CAAAA,yBAAAA,QAAQ,aAAa,AAAD,IAApBA,KAAAA,IAAAA,uBAAuB,aAAa,EACtCtE,KAAK,IAAI,CAAC;QACR,MAAM;QACN,SAAS;IACX;IAGF,IAAIF,kBAAkB;QACpB,MAAM0B,SAAS,MAAM3B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAIwB;IACf;IAEA,MAAM+B,SAAS,MAAMC,SACnBxD,MACA4B,aAAa,YAAY,EACzBlB;IAEF,OAAO;QACL,aAAa6C,OAAO,OAAO;QAC3B3C;QACA,OAAO2C,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/llm-planning.mjs

@@ -76,7 +76,6 @@ async function plan(userInstruction, opts) {
                 const element = elementById(locateResult);
                 if (element) action.param[field].id = element.id;
             }
-            action.locate = action.param[field];
         });
     });
     assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);

package/dist/es/ai-model/llm-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/llm-planning.mjs","sources":["webpack://@midscene/core/./src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n  DeviceAction,\n  PageType,\n  PlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport { type IModelPreferences, vlLocateMode } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport {\n  AIActionType,\n  type AIArgs,\n  buildYamlFlowFromPlans,\n  callAiFn,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n  markupImageForLLM,\n  warnGPT4oSizeLimit,\n} from './common';\nimport {\n  automationUserPrompt,\n  generateTaskBackgroundContext,\n  systemPromptToTaskPlanning,\n} from './prompt/llm-planning';\nimport { describeUserPage } from './prompt/util';\n\nconst debug = getDebug('planning');\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    pageType: PageType;\n    actionSpace: DeviceAction<any>[];\n    callAI?: typeof callAiFn<PlanningAIResponse>;\n    log?: string;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { callAI, context } = opts || {};\n  const { screenshotBase64, size } = context;\n\n  const modelPreferences: IModelPreferences = {\n    intent: 'planning',\n  };\n  const { description: pageDescription, elementById } = await describeUserPage(\n    context,\n    modelPreferences,\n  );\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode: vlLocateMode(modelPreferences),\n  });\n  const taskBackgroundContextText = generateTaskBackgroundContext(\n    userInstruction,\n    opts.log,\n    opts.actionContext,\n  );\n  const userInstructionPrompt = await automationUserPrompt(\n    vlLocateMode(modelPreferences),\n  ).format({\n    pageDescription,\n    taskBackgroundContext: taskBackgroundContextText,\n  });\n\n  let imagePayload = screenshotBase64;\n  if (vlLocateMode(modelPreferences) === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlLocateMode(modelPreferences)) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  warnGPT4oSizeLimit(size, modelPreferences);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  const call = callAI || callAiFn;\n  const { content, usage } = await call(\n    msgs,\n    AIActionType.PLAN,\n    modelPreferences,\n  );\n  const rawResponse = JSON.stringify(content, undefined, 2);\n  const planFromAI = content;\n\n  const actions =\n    (planFromAI.action?.type ? [planFromAI.action] : planFromAI.actions) || [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: use zod.parse to parse the action.param, and then fill the bbox param.\n  actions.forEach((action) => {\n    const type = action.type;\n    const actionInActionSpace = opts.actionSpace.find(\n      (action) => action.name === type,\n    );\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param[field];\n      if (locateResult) {\n        if (vlLocateMode(modelPreferences)) {\n          action.param[field] = fillBboxParam(\n            locateResult,\n            size.width,\n            size.height,\n            modelPreferences,\n          );\n        } else {\n          const element = elementById(locateResult);\n          if (element) {\n            action.param[field].id = element.id;\n          }\n        }\n      }\n\n      // to be compatible with the web-integration\n      action.locate = action.param[field];\n    });\n  });\n  // in Qwen-VL, error means error. In GPT-4o, error may mean more actions are needed.\n  assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","plan","userInstruction","opts","_planFromAI_action","callAI","context","screenshotBase64","size","modelPreferences","pageDescription","elementById","describeUserPage","systemPrompt","systemPromptToTaskPlanning","vlLocateMode","taskBackgroundContextText","generateTaskBackgroundContext","userInstructionPrompt","automationUserPrompt","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","warnGPT4oSizeLimit","msgs","call","callAiFn","content","usage","AIActionType","rawResponse","JSON","undefined","planFromAI","actions","returnValue","buildYamlFlowFromPlans","assert","action","type","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","element","console"],"mappings":";;;;;;;AA2BA,MAAMA,QAAQC,SAAS;AAEhB,eAAeC,KACpBC,eAAuB,EACvBC,IAOC;QAwEEC;IAtEH,MAAM,EAAEC,MAAM,EAAEC,OAAO,EAAE,GAAGH,QAAQ,CAAC;IACrC,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGF;IAEnC,MAAMG,mBAAsC;QAC1C,QAAQ;IACV;IACA,MAAM,EAAE,aAAaC,eAAe,EAAEC,WAAW,EAAE,GAAG,MAAMC,iBAC1DN,SACAG;IAGF,MAAMI,eAAe,MAAMC,2BAA2B;QACpD,aAAaX,KAAK,WAAW;QAC7B,QAAQY,aAAaN;IACvB;IACA,MAAMO,4BAA4BC,8BAChCf,iBACAC,KAAK,GAAG,EACRA,KAAK,aAAa;IAEpB,MAAMe,wBAAwB,MAAMC,qBAClCJ,aAAaN,mBACb,MAAM,CAAC;QACPC;QACA,uBAAuBM;IACzB;IAEA,IAAII,eAAeb;IACnB,IAAIQ,AAAmC,cAAnCA,aAAaN,mBACfW,eAAe,MAAMC,4BAA4BD;SAC5C,IAAI,CAACL,aAAaN,mBACvBW,eAAe,MAAME,kBACnBf,kBACAD,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhBiB,mBAAmBf,MAAMC;IAEzB,MAAMe,OAAe;QACnB;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKO;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMF;gBACR;aACD;QACH;KACD;IAED,MAAMO,OAAOpB,UAAUqB;IACvB,MAAM,EAAEC,OAAO,EAAEC,KAAK,EAAE,GAAG,MAAMH,KAC/BD,MACAK,aAAa,IAAI,EACjBpB;IAEF,MAAMqB,cAAcC,KAAK,SAAS,CAACJ,SAASK,QAAW;IACvD,MAAMC,aAAaN;IAEnB,MAAMO,UACH9B,AAAAA,CAAAA,SAAAA,CAAAA,qBAAAA,WAAW,MAAM,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,IAAI;QAAC6B,WAAW,MAAM;KAAC,GAAGA,WAAW,OAAM,KAAM,EAAE;IAC5E,MAAME,cAAkC;QACtC,GAAGF,UAAU;QACbC;QACAJ;QACAF;QACA,UAAUQ,uBACRF,SACA/B,KAAK,WAAW,EAChB8B,WAAW,KAAK;IAEpB;IAEAI,OAAOJ,YAAY;IAGnBC,QAAQ,OAAO,CAAC,CAACI;QACf,MAAMC,OAAOD,OAAO,IAAI;QACxB,MAAME,sBAAsBrC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACmC,SAAWA,OAAO,IAAI,KAAKC;QAE9B,MAAME,eAAeD,sBACjBE,4BAA4BF,oBAAoB,WAAW,IAC3D,EAAE;QAENzC,MAAM,gBAAgB0C;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,CAACK,MAAM;YACxC,IAAIC,cACF,IAAI7B,aAAaN,mBACf6B,OAAO,KAAK,CAACK,MAAM,GAAGE,cACpBD,cACApC,KAAK,KAAK,EACVA,KAAK,MAAM,EACXC;iBAEG;gBACL,MAAMqC,UAAUnC,YAAYiC;gBAC5B,IAAIE,SACFR,OAAO,KAAK,CAACK,MAAM,CAAC,EAAE,GAAGG,QAAQ,EAAE;YAEvC;YAIFR,OAAO,MAAM,GAAGA,OAAO,KAAK,CAACK,MAAM;QACrC;IACF;IAEAN,OAAO,CAACJ,WAAW,KAAK,EAAE,CAAC,wBAAwB,EAAEA,WAAW,KAAK,EAAE;IAEvE,IACEC,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBY,QAAQ,IAAI,CACV,8EACA7C;IAIJ,OAAOiC;AACT"}
+{"version":3,"file":"ai-model/llm-planning.mjs","sources":["webpack://@midscene/core/./src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n  DeviceAction,\n  PageType,\n  PlanningAIResponse,\n  UIContext,\n} from '@/types';\nimport { type IModelPreferences, vlLocateMode } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport {\n  AIActionType,\n  type AIArgs,\n  buildYamlFlowFromPlans,\n  callAiFn,\n  fillBboxParam,\n  findAllMidsceneLocatorField,\n  markupImageForLLM,\n  warnGPT4oSizeLimit,\n} from './common';\nimport {\n  automationUserPrompt,\n  generateTaskBackgroundContext,\n  systemPromptToTaskPlanning,\n} from './prompt/llm-planning';\nimport { describeUserPage } from './prompt/util';\n\nconst debug = getDebug('planning');\n\nexport async function plan(\n  userInstruction: string,\n  opts: {\n    context: UIContext;\n    pageType: PageType;\n    actionSpace: DeviceAction<any>[];\n    callAI?: typeof callAiFn<PlanningAIResponse>;\n    log?: string;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { callAI, context } = opts || {};\n  const { screenshotBase64, size } = context;\n\n  const modelPreferences: IModelPreferences = {\n    intent: 'planning',\n  };\n  const { description: pageDescription, elementById } = await describeUserPage(\n    context,\n    modelPreferences,\n  );\n\n  const systemPrompt = await systemPromptToTaskPlanning({\n    actionSpace: opts.actionSpace,\n    vlMode: vlLocateMode(modelPreferences),\n  });\n  const taskBackgroundContextText = generateTaskBackgroundContext(\n    userInstruction,\n    opts.log,\n    opts.actionContext,\n  );\n  const userInstructionPrompt = await automationUserPrompt(\n    vlLocateMode(modelPreferences),\n  ).format({\n    pageDescription,\n    taskBackgroundContext: taskBackgroundContextText,\n  });\n\n  let imagePayload = screenshotBase64;\n  if (vlLocateMode(modelPreferences) === 'qwen-vl') {\n    imagePayload = await paddingToMatchBlockByBase64(imagePayload);\n  } else if (!vlLocateMode(modelPreferences)) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  warnGPT4oSizeLimit(size, modelPreferences);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  const call = callAI || callAiFn;\n  const { content, usage } = await call(\n    msgs,\n    AIActionType.PLAN,\n    modelPreferences,\n  );\n  const rawResponse = JSON.stringify(content, undefined, 2);\n  const planFromAI = content;\n\n  const actions =\n    (planFromAI.action?.type ? [planFromAI.action] : planFromAI.actions) || [];\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    usage,\n    yamlFlow: buildYamlFlowFromPlans(\n      actions,\n      opts.actionSpace,\n      planFromAI.sleep,\n    ),\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: use zod.parse to parse the action.param, and then fill the bbox param.\n  actions.forEach((action) => {\n    const type = action.type;\n    const actionInActionSpace = opts.actionSpace.find(\n      (action) => action.name === type,\n    );\n    const locateFields = actionInActionSpace\n      ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n      : [];\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const locateResult = action.param[field];\n      if (locateResult) {\n        if (vlLocateMode(modelPreferences)) {\n          action.param[field] = fillBboxParam(\n            locateResult,\n            size.width,\n            size.height,\n            modelPreferences,\n          );\n        } else {\n          const element = elementById(locateResult);\n          if (element) {\n            action.param[field].id = element.id;\n          }\n        }\n      }\n    });\n  });\n  // in Qwen-VL, error means error. In GPT-4o, error may mean more actions are needed.\n  assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);\n\n  if (\n    actions.length === 0 &&\n    returnValue.more_actions_needed_by_instruction &&\n    !returnValue.sleep\n  ) {\n    console.warn(\n      'No actions planned for the prompt, but model said more actions are needed:',\n      userInstruction,\n    );\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","plan","userInstruction","opts","_planFromAI_action","callAI","context","screenshotBase64","size","modelPreferences","pageDescription","elementById","describeUserPage","systemPrompt","systemPromptToTaskPlanning","vlLocateMode","taskBackgroundContextText","generateTaskBackgroundContext","userInstructionPrompt","automationUserPrompt","imagePayload","paddingToMatchBlockByBase64","markupImageForLLM","warnGPT4oSizeLimit","msgs","call","callAiFn","content","usage","AIActionType","rawResponse","JSON","undefined","planFromAI","actions","returnValue","buildYamlFlowFromPlans","assert","action","type","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","element","console"],"mappings":";;;;;;;AA2BA,MAAMA,QAAQC,SAAS;AAEhB,eAAeC,KACpBC,eAAuB,EACvBC,IAOC;QAwEEC;IAtEH,MAAM,EAAEC,MAAM,EAAEC,OAAO,EAAE,GAAGH,QAAQ,CAAC;IACrC,MAAM,EAAEI,gBAAgB,EAAEC,IAAI,EAAE,GAAGF;IAEnC,MAAMG,mBAAsC;QAC1C,QAAQ;IACV;IACA,MAAM,EAAE,aAAaC,eAAe,EAAEC,WAAW,EAAE,GAAG,MAAMC,iBAC1DN,SACAG;IAGF,MAAMI,eAAe,MAAMC,2BAA2B;QACpD,aAAaX,KAAK,WAAW;QAC7B,QAAQY,aAAaN;IACvB;IACA,MAAMO,4BAA4BC,8BAChCf,iBACAC,KAAK,GAAG,EACRA,KAAK,aAAa;IAEpB,MAAMe,wBAAwB,MAAMC,qBAClCJ,aAAaN,mBACb,MAAM,CAAC;QACPC;QACA,uBAAuBM;IACzB;IAEA,IAAII,eAAeb;IACnB,IAAIQ,AAAmC,cAAnCA,aAAaN,mBACfW,eAAe,MAAMC,4BAA4BD;SAC5C,IAAI,CAACL,aAAaN,mBACvBW,eAAe,MAAME,kBACnBf,kBACAD,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhBiB,mBAAmBf,MAAMC;IAEzB,MAAMe,OAAe;QACnB;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKO;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMF;gBACR;aACD;QACH;KACD;IAED,MAAMO,OAAOpB,UAAUqB;IACvB,MAAM,EAAEC,OAAO,EAAEC,KAAK,EAAE,GAAG,MAAMH,KAC/BD,MACAK,aAAa,IAAI,EACjBpB;IAEF,MAAMqB,cAAcC,KAAK,SAAS,CAACJ,SAASK,QAAW;IACvD,MAAMC,aAAaN;IAEnB,MAAMO,UACH9B,AAAAA,CAAAA,SAAAA,CAAAA,qBAAAA,WAAW,MAAM,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,IAAI;QAAC6B,WAAW,MAAM;KAAC,GAAGA,WAAW,OAAM,KAAM,EAAE;IAC5E,MAAME,cAAkC;QACtC,GAAGF,UAAU;QACbC;QACAJ;QACAF;QACA,UAAUQ,uBACRF,SACA/B,KAAK,WAAW,EAChB8B,WAAW,KAAK;IAEpB;IAEAI,OAAOJ,YAAY;IAGnBC,QAAQ,OAAO,CAAC,CAACI;QACf,MAAMC,OAAOD,OAAO,IAAI;QACxB,MAAME,sBAAsBrC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACmC,SAAWA,OAAO,IAAI,KAAKC;QAE9B,MAAME,eAAeD,sBACjBE,4BAA4BF,oBAAoB,WAAW,IAC3D,EAAE;QAENzC,MAAM,gBAAgB0C;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,eAAeN,OAAO,KAAK,CAACK,MAAM;YACxC,IAAIC,cACF,IAAI7B,aAAaN,mBACf6B,OAAO,KAAK,CAACK,MAAM,GAAGE,cACpBD,cACApC,KAAK,KAAK,EACVA,KAAK,MAAM,EACXC;iBAEG;gBACL,MAAMqC,UAAUnC,YAAYiC;gBAC5B,IAAIE,SACFR,OAAO,KAAK,CAACK,MAAM,CAAC,EAAE,GAAGG,QAAQ,EAAE;YAEvC;QAEJ;IACF;IAEAT,OAAO,CAACJ,WAAW,KAAK,EAAE,CAAC,wBAAwB,EAAEA,WAAW,KAAK,EAAE;IAEvE,IACEC,AAAmB,MAAnBA,QAAQ,MAAM,IACdC,YAAY,kCAAkC,IAC9C,CAACA,YAAY,KAAK,EAElBY,QAAQ,IAAI,CACV,8EACA7C;IAIJ,OAAOiC;AACT"}

package/dist/es/ai-model/prompt/llm-planning.mjs

@@ -19,23 +19,42 @@ const descriptionForAction = (action, locatorSchemaTypeDescription)=>{
         const shape = action.paramSchema.shape;
         const paramLines = [];
         const getTypeName = (field)=>{
-            var _field__def, _actualField__def, _actualField__def1, _actualField__def2, _actualField__def3, _actualField__def4;
-            const actualField = (null == (_field__def = field._def) ? void 0 : _field__def.innerType) || field;
-            if ((null == (_actualField__def = actualField._def) ? void 0 : _actualField__def.typeName) === 'ZodString') return 'string';
-            if ((null == (_actualField__def1 = actualField._def) ? void 0 : _actualField__def1.typeName) === 'ZodNumber') return 'number';
-            if ((null == (_actualField__def2 = actualField._def) ? void 0 : _actualField__def2.typeName) === 'ZodBoolean') return 'boolean';
-            if ((null == (_actualField__def3 = actualField._def) ? void 0 : _actualField__def3.typeName) === 'ZodArray') return 'array';
-            if ((null == (_actualField__def4 = actualField._def) ? void 0 : _actualField__def4.typeName) === 'ZodObject') {
+            var _actualField__def;
+            const unwrapField = (f)=>{
+                if (!f._def) return f;
+                const typeName = f._def.typeName;
+                if ('ZodOptional' === typeName || 'ZodNullable' === typeName || 'ZodDefault' === typeName) return unwrapField(f._def.innerType);
+                return f;
+            };
+            const actualField = unwrapField(field);
+            const fieldTypeName = null == (_actualField__def = actualField._def) ? void 0 : _actualField__def.typeName;
+            if ('ZodString' === fieldTypeName) return 'string';
+            if ('ZodNumber' === fieldTypeName) return 'number';
+            if ('ZodBoolean' === fieldTypeName) return 'boolean';
+            if ('ZodArray' === fieldTypeName) return 'array';
+            if ('ZodObject' === fieldTypeName) {
                 if (ifMidsceneLocatorField(actualField)) return locatorSchemaTypeDescription;
                 return 'object';
             }
-            console.warn('unknown type: ', actualField._def);
-            return 'type';
+            if ('ZodEnum' === fieldTypeName) {
+                var _actualField__def_values, _actualField__def1;
+                const values = (null == (_actualField__def1 = actualField._def) ? void 0 : null == (_actualField__def_values = _actualField__def1.values) ? void 0 : _actualField__def_values.map((option)=>String(`'${option}'`)).join(', ')) ?? 'enum';
+                return `enum(${values})`;
+            }
+            console.warn('failed to parse Zod type. This may lead to wrong params from the LLM.\n', actualField._def);
+            return actualField.toString();
         };
         const getDescription = (field)=>{
-            var _field__def, _actualField__def;
-            const actualField = (null == (_field__def = field._def) ? void 0 : _field__def.innerType) || field;
+            var _actualField__def;
+            const unwrapField = (f)=>{
+                if (!f._def) return f;
+                const typeName = f._def.typeName;
+                if ('ZodOptional' === typeName || 'ZodNullable' === typeName || 'ZodDefault' === typeName) return unwrapField(f._def.innerType);
+                return f;
+            };
             if ("description" in field) return field.description || null;
+            const actualField = unwrapField(field);
+            if ("description" in actualField) return actualField.description || null;
             if ((null == (_actualField__def = actualField._def) ? void 0 : _actualField__def.typeName) === 'ZodObject') {
                 if ('midscene_location_field_flag' in actualField._def.shape()) return 'Location information for the target element';
             }
@@ -101,9 +120,11 @@ this and output the JSON:
   "more_actions_needed_by_instruction": false,
   "action": {
     "type": "Tap",
-    "locate": {
-      "bbox": [100, 100, 200, 200],
-      "prompt": "The 'Yes' button in popup"
+    "param": {
+      "locate": {
+        "bbox": [100, 100, 200, 200],
+        "prompt": "The 'Yes' button in popup"
+      }
     }
   }
 }
@@ -193,8 +214,9 @@ By viewing the page screenshot and description, you should consider this and out
     {
       "thought": "Click the language switch button to open the language options.",
       "type": "Tap", 
-      "param": null,
-      "locate": { id: "c81c4e9a33", prompt: "The language switch button" }},
+      "param": {
+        "locate": { id: "c81c4e9a33", prompt: "The language switch button" }
+      }
     },
     {
       "thought": "Wait for 1 second to ensure the language options are displayed.",
@@ -214,16 +236,16 @@ Wrong output:
     {
       "thought": "Click the language switch button to open the language options.",
       "type": "Tap",
-      "param": null,
-      "locate": {
-        { "id": "c81c4e9a33" }, // WRONG: prompt is missing, this is not a valid LocateParam
+      "param": {
+        "locate": { "id": "c81c4e9a33" } // WRONG: prompt is missing, this is not a valid LocateParam
       }
     },
     {
       "thought": "Click the English option",
       "type": "Tap", 
-      "param": null,
-      "locate": null, // This means the 'English' option is not shown in the screenshot, the task cannot be accomplished
+      "param": {
+        "locate": null // WRONG: if the element is not on the page, you should not plan this action
+      }
     }
   ],
   "more_actions_needed_by_instruction": false, // WRONG: should be true