@midscene/core 1.0.1-beta-20251024063839.0 → 1.0.1-beta-20251027033034.0

package/dist/es/ai-model/inspect.mjs

@@ -1,3 +1,4 @@
+import { generateElementByPosition } from "@midscene/shared/extractor/dom-util";
 import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
@@ -5,7 +6,7 @@ import { AIActionType, adaptBboxToRect, expandSearchArea, mergeRects } from "./c
 import { extractDataQueryPrompt, systemPromptToExtract } from "./prompt/extraction.mjs";
 import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
-import { describeUserPage, distance, distanceThreshold, elementByPositionWithElementInfo } from "./prompt/util.mjs";
+import { describeUserPage } from "./prompt/util.mjs";
 import { callAIWithObjectResponse } from "./service-caller/index.mjs";
 const debugInspect = getDebug('ai:inspect');
 const debugSection = getDebug('ai:section');
@@ -57,12 +58,9 @@ async function AiLocateElement(options) {
     const { context, targetElementDescription, callAIFn, modelConfig } = options;
     const { vlMode } = modelConfig;
     const { screenshotBase64 } = context;
-    const { description, elementById, insertElementByPosition } = await describeUserPage(context, {
-        vlMode
-    });
     assert(targetElementDescription, "cannot find the target element description");
     const userInstructionPrompt = await findElementPrompt.format({
-        pageDescription: description,
+        pageDescription: await describeUserPage(context),
         targetElementDescription: extraTextFromUserPrompt(targetElementDescription)
     });
     const systemPrompt = systemPromptToLocateElement(vlMode);
@@ -129,18 +127,12 @@ async function AiLocateElement(options) {
                 x: resRect.left + resRect.width / 2,
                 y: resRect.top + resRect.height / 2
             };
-            let element = elementByPositionWithElementInfo(context.tree, rectCenter);
-            const distanceToCenter = element ? distance({
-                x: element.center[0],
-                y: element.center[1]
-            }, rectCenter) : 0;
-            if (!element || distanceToCenter > distanceThreshold) element = insertElementByPosition(rectCenter);
-            if (element) {
-                matchedElements = [
-                    element
-                ];
-                errors = [];
-            }
+            const element = generateElementByPosition(rectCenter);
+            errors = [];
+            element.isOrderSensitive = 'object' == typeof res.content && null !== res.content && 'isOrderSensitive' in res.content ? res.content.isOrderSensitive : void 0;
+            if (element) matchedElements = [
+                element
+            ];
         }
     } catch (e) {
         const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
@@ -153,12 +145,10 @@ async function AiLocateElement(options) {
         rect: resRect,
         parseResult: {
             elements: matchedElements,
-            errors
+            errors: errors
         },
         rawResponse,
-        elementById,
-        usage: res.usage,
-        isOrderSensitive: 'object' == typeof res.content && null !== res.content && 'isOrderSensitive' in res.content ? res.content.isOrderSensitive : void 0
+        usage: res.usage
     };
 }
 async function AiLocateSection(options) {
@@ -233,17 +223,9 @@ async function AiLocateSection(options) {
 }
 async function AiExtractElementInfo(options) {
     const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } = options;
-    const { vlMode } = modelConfig;
     const systemPrompt = systemPromptToExtract();
     const { screenshotBase64 } = context;
-    const { description, elementById } = await describeUserPage(context, {
-        truncateTextLength: 200,
-        filterNonTextContent: false,
-        visibleOnly: false,
-        domIncluded: null == extractOption ? void 0 : extractOption.domIncluded,
-        vlMode
-    });
-    const extractDataPromptText = await extractDataQueryPrompt(description, dataQuery);
+    const extractDataPromptText = await extractDataQueryPrompt(options.pageDescription || '', dataQuery);
     const userContent = [];
     if ((null == extractOption ? void 0 : extractOption.screenshotIncluded) !== false) userContent.push({
         type: 'image_url',
@@ -276,7 +258,6 @@ async function AiExtractElementInfo(options) {
     const result = await callAIWithObjectResponse(msgs, AIActionType.EXTRACT_DATA, modelConfig);
     return {
         parseResult: result.content,
-        elementById,
         usage: result.usage
     };
 }

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocatorResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  BaseElement,\n  ElementById,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from './common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  mergeRects,\n} from './common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  describeUserPage,\n  distance,\n  distanceThreshold,\n  elementByPositionWithElementInfo,\n} from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement<\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  context: UIContext<ElementType>;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: AIElementLocatorResponse;\n  rect?: Rect;\n  rawResponse: string;\n  elementById: ElementById;\n  usage?: AIUsageInfo;\n  isOrderSensitive?: boolean;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const { description, elementById, insertElementByPosition } =\n    await describeUserPage(context, { vlMode });\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: description,\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: AIElementLocatorResponse['elements'] =\n    'elements' in res.content ? res.content.elements : [];\n  let errors: AIElementLocatorResponse['errors'] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n      let element = elementByPositionWithElementInfo(context.tree, rectCenter);\n\n      const distanceToCenter = element\n        ? distance({ x: element.center[0], y: element.center[1] }, rectCenter)\n        : 0;\n\n      if (!element || distanceToCenter > distanceThreshold) {\n        element = insertElementByPosition(rectCenter);\n      }\n\n      if (element) {\n        matchedElements = [element];\n        errors = [];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements,\n      errors,\n    },\n    rawResponse,\n    elementById,\n    usage: res.usage,\n    isOrderSensitive:\n      typeof res.content === 'object' &&\n      res.content !== null &&\n      'isOrderSensitive' in res.content\n        ? (res.content as any).isOrderSensitive\n        : undefined,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext<BaseElement>;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<\n  T,\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext<ElementType>;\n  extractOption?: InsightExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const { vlMode } = modelConfig;\n  const systemPrompt = systemPromptToExtract();\n\n  const { screenshotBase64 } = context;\n\n  const { description, elementById } = await describeUserPage(context, {\n    truncateTextLength: 200,\n    filterNonTextContent: false,\n    visibleOnly: false,\n    domIncluded: extractOption?.domIncluded,\n    vlMode,\n  });\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    description,\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    elementById,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","description","elementById","insertElementByPosition","describeUserPage","assert","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","_options_searchConfig_rect","_options_searchConfig_rect1","paddedResult","paddingToMatchBlockByBase64","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","elementByPositionWithElementInfo","distanceToCenter","distance","distanceThreshold","e","msg","Error","undefined","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent"],"mappings":";;;;;;;;;AAyDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAEpBC,OASD;IAQC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAM,EAAEM,WAAW,EAAEC,WAAW,EAAEC,uBAAuB,EAAE,GACzD,MAAMC,iBAAiBT,SAAS;QAAEI;IAAO;IAE3CM,OACET,0BACA;IAEF,MAAMU,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiBN;QACjB,0BAA0BjB,wBAAwBY;IACpD;IACA,MAAMY,eAAeC,4BAA4BV;IAEjD,IAAIW,eAAeV;IACnB,IAAIW,aAAahB,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIiB,cAAcjB,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIkB,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIlB,QAAQ,YAAY,EAAE;YAWXqB,4BACCC;QAXdX,OACEX,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFW,OACEX,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFgB,eAAehB,QAAQ,YAAY,CAAC,WAAW;QAC/CiB,aAAa,QAAAI,CAAAA,6BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,KAAK;QAC7CH,cAAc,QAAAI,CAAAA,8BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,MAAM;QAC/CH,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIb,AAAW,cAAXA,QAAsB;QAC/B,MAAMkB,eAAe,MAAMC,4BAA4BR;QACvDC,aAAaM,aAAa,KAAK;QAC/BL,cAAcK,aAAa,MAAM;QACjCP,eAAeO,aAAa,WAAW;IACzC;IAEA,MAAM5B,OAAe;QACnB;YAAE,MAAM;YAAU,SAASmB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMJ;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOV,0BAAuC;QAChD,MAAMuB,SAAS,MAAMjC,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI8B;IACf;IAEA,MAAMC,MAAM,MAAMvB,SAASR,MAAMgC,aAAa,eAAe,EAAEvB;IAE/D,MAAMwB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBACF,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IACvD,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAK1DQ,6BAAAA,uBACAC,6BAAAA;YALFL,UAAUM,gBACRV,IAAI,OAAO,CAAC,IAAI,EAChBT,YACAC,aAAAA,QACAgB,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG,EAC/BhB,oBACAC,qBACAf;YAGFlB,aAAa,WAAW2C;YAExB,MAAMO,aAAa;gBACjB,GAAGP,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YACA,IAAIQ,UAAUC,iCAAiCtC,QAAQ,IAAI,EAAEoC;YAE7D,MAAMG,mBAAmBF,UACrBG,SAAS;gBAAE,GAAGH,QAAQ,MAAM,CAAC,EAAE;gBAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;YAAC,GAAGD,cACzD;YAEJ,IAAI,CAACC,WAAWE,mBAAmBE,mBACjCJ,UAAU7B,wBAAwB4B;YAGpC,IAAIC,SAAS;gBACXP,kBAAkB;oBAACO;iBAAQ;gBAC3BN,SAAS,EAAE;YACb;QACF;IACF,EAAE,OAAOW,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACX,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEY,IAAI,CAAC,CAAC;aAFtBZ,SAAS;YAACY;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMd;QACN,aAAa;YACX,UAAUC;YACVC;QACF;QACAJ;QACApB;QACA,OAAOkB,IAAI,KAAK;QAChB,kBACE,AAAuB,YAAvB,OAAOA,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCoB;IACR;AACF;AAEO,eAAeC,gBAAgB/C,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAE+C,kBAAkB,EAAE5C,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMa,eAAemC,4BAA4B5C;IACjD,MAAM6C,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoB7D,wBAAwB0D;IAC9C;IACA,MAAMrD,OAAe;QACnB;YAAE,MAAM;YAAU,SAASmB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM4C;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMvB,SAAS,MAAMjC,mBAAmB;YACtC,QAAQwD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACArD,KAAK,IAAI,IAAI8B;IACf;IAEA,MAAM2B,SAAS,MAAMC,yBACnB1D,MACAgC,aAAa,YAAY,EACzBvB;IAGF,IAAIkD;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAapB,gBACjBmB,aACAtD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0BmE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9D/D,aAAa,wBAAwBoE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAS1B,MAAM,OAAO,CAAC0B,OAC/B,GAAG,CAAC,CAACA,OACGvB,gBACLuB,MACA1D,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqBqE;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DrE,aAAa,iBAAiBuE;QAG9BN,cAAcQ,iBAAiBF,YAAY3D,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2BiE;IAC1C;IAEA,IAAIS,cAAczD;IAClB,IAAIgD,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1B3D,kBACAgD,aACAjD,AAAW,cAAXA;QAEF0D,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAavB,KAAK,SAAS,CAACuB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAGpBlE,OAMD;IACC,MAAM,EAAEmE,SAAS,EAAElE,OAAO,EAAEmE,aAAa,EAAE3E,gBAAgB,EAAEW,WAAW,EAAE,GACxEJ;IACF,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAMU,eAAeuD;IAErB,MAAM,EAAE/D,gBAAgB,EAAE,GAAGL;IAE7B,MAAM,EAAEM,WAAW,EAAEC,WAAW,EAAE,GAAG,MAAME,iBAAiBT,SAAS;QACnE,oBAAoB;QACpB,sBAAsB;QACtB,aAAa;QACb,aAAamE,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,WAAW;QACvC/D;IACF;IAEA,MAAMiE,wBAAwB,MAAMC,uBAClChE,aACA4D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKlE;YACL,QAAQ;QACV;IACF;IAGFkE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM3E,OAAe;QACnB;YAAE,MAAM;YAAU,SAASmB;QAAa;QACxC;YACE,MAAM;YACN,SAAS0D;QACX;KACD;IAED,IAAI/E,kBAAkB;QACpB,MAAMgC,SAAS,MAAMjC,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAI8B;IACf;IAEA,MAAM2B,SAAS,MAAMC,yBACnB1D,MACAgC,aAAa,YAAY,EACzBvB;IAEF,OAAO;QACL,aAAagD,OAAO,OAAO;QAC3B5C;QACA,OAAO4C,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from './common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  mergeRects,\n} from './common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport { describeUserPage } from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const userInstructionPrompt = await findElementPrompt.format({\n    pageDescription: await describeUserPage(context),\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements = 'elements' in res.content ? res.content.elements : [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement =\n        generateElementByPosition(rectCenter);\n      errors = [];\n\n      element.isOrderSensitive =\n        typeof res.content === 'object' &&\n        res.content !== null &&\n        'isOrderSensitive' in res.content\n          ? (res.content as any).isOrderSensitive\n          : undefined;\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: InsightExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = await extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","userInstructionPrompt","findElementPrompt","describeUserPage","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","_options_searchConfig_rect","_options_searchConfig_rect1","paddedResult","paddingToMatchBlockByBase64","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","generateElementByPosition","undefined","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent"],"mappings":";;;;;;;;;;AAmDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IASC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,wBAAwB,MAAMC,kBAAkB,MAAM,CAAC;QAC3D,iBAAiB,MAAMC,iBAAiBT;QACxC,0BAA0BX,wBAAwBY;IACpD;IACA,MAAMS,eAAeC,4BAA4BP;IAEjD,IAAIQ,eAAeP;IACnB,IAAIQ,aAAab,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIc,cAAcd,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIe,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIf,QAAQ,YAAY,EAAE;YAWXkB,4BACCC;QAXdZ,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFa,eAAeb,QAAQ,YAAY,CAAC,WAAW;QAC/Cc,aAAa,QAAAI,CAAAA,6BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,KAAK;QAC7CH,cAAc,QAAAI,CAAAA,8BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,MAAM;QAC/CH,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIV,AAAW,cAAXA,QAAsB;QAC/B,MAAMe,eAAe,MAAMC,4BAA4BR;QACvDC,aAAaM,aAAa,KAAK;QAC/BL,cAAcK,aAAa,MAAM;QACjCP,eAAeO,aAAa,WAAW;IACzC;IAEA,MAAMzB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAML;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAON,0BAAuC;QAChD,MAAMoB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMC,MAAM,MAAMpB,SAASR,MAAM6B,aAAa,eAAe,EAAEpB;IAE/D,MAAMqB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBAAkB,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IAC3E,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAK1DQ,6BAAAA,uBACAC,6BAAAA;YALFL,UAAUM,gBACRV,IAAI,OAAO,CAAC,IAAI,EAChBT,YACAC,aAAAA,QACAgB,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG,EAC/BhB,oBACAC,qBACAZ;YAGFlB,aAAa,WAAWwC;YAExB,MAAMO,aAAa;gBACjB,GAAGP,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMQ,UACJC,0BAA0BF;YAC5BL,SAAS,EAAE;YAEXM,QAAQ,gBAAgB,GACtB,AAAuB,YAAvB,OAAOZ,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCc;YAEN,IAAIF,SACFP,kBAAkB;gBAACO;aAAQ;QAE/B;IACF,EAAE,OAAOG,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACT,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEU,IAAI,CAAC,CAAC;aAFtBV,SAAS;YAACU;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMZ;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAJ;QACA,OAAOF,IAAI,KAAK;IAClB;AACF;AAEO,eAAekB,gBAAgBzC,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEyC,kBAAkB,EAAEtC,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAegC,4BAA4BtC;IACjD,MAAMuC,gCAAgC,MAAMC,0BAA0B,MAAM,CAAC;QAC3E,oBAAoBvD,wBAAwBoD;IAC9C;IACA,MAAM/C,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMsC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMpB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQkD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA/C,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMwB,SAAS,MAAMC,yBACnBpD,MACA6B,aAAa,YAAY,EACzBpB;IAGF,IAAI4C;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAajB,gBACjBgB,aACAhD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0B6D;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DzD,aAAa,wBAAwB8D;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASvB,MAAM,OAAO,CAACuB,OAC/B,GAAG,CAAC,CAACA,OACGpB,gBACLoB,MACApD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqB+D;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7D/D,aAAa,iBAAiBiE;QAG9BN,cAAcQ,iBAAiBF,YAAYrD,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2B2D;IAC1C;IAEA,IAAIS,cAAcnD;IAClB,IAAI0C,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1BrD,kBACA0C,aACA3C,AAAW,cAAXA;QAEFoD,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAapB,KAAK,SAAS,CAACoB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwB5D,OAO7C;IACC,MAAM,EAAE6D,SAAS,EAAE5D,OAAO,EAAE6D,aAAa,EAAErE,gBAAgB,EAAEW,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAeoD;IACrB,MAAM,EAAEzD,gBAAgB,EAAE,GAAGL;IAE7B,MAAM+D,wBAAwB,MAAMC,uBAClCjE,QAAQ,eAAe,IAAI,IAC3B6D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAK5D;YACL,QAAQ;QACV;IACF;IAGF4D,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMrE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASuD;QACX;KACD;IAED,IAAIzE,kBAAkB;QACpB,MAAM6B,SAAS,MAAM9B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMwB,SAAS,MAAMC,yBACnBpD,MACA6B,aAAa,YAAY,EACzBpB;IAEF,OAAO;QACL,aAAa0C,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/prompt/util.mjs

@@ -1,7 +1,3 @@
-import { imageInfoOfBase64 } from "../../image/index.mjs";
-import { NodeType } from "@midscene/shared/constants";
-import { descriptionOfTree, generateElementByPosition, treeToList } from "@midscene/shared/extractor";
-import { assert } from "@midscene/shared/utils";
 function describeSize(size) {
     return `${size.width} x ${size.height}`;
 }
@@ -17,35 +13,6 @@ function describeElement(elements) {
         ].join(', ')).join('\n');
 }
 const distanceThreshold = 16;
-function elementByPositionWithElementInfo(treeRoot, position, options) {
-    const requireStrictDistance = (null == options ? void 0 : options.requireStrictDistance) ?? true;
-    const filterPositionElements = (null == options ? void 0 : options.filterPositionElements) ?? false;
-    assert(void 0 !== position, 'position is required for query');
-    const matchingElements = [];
-    function dfs(node) {
-        if (null == node ? void 0 : node.node) {
-            const item = node.node;
-            if (item.rect.left <= position.x && position.x <= item.rect.left + item.rect.width && item.rect.top <= position.y && position.y <= item.rect.top + item.rect.height) {
-                var _item_attributes;
-                if (!(filterPositionElements && (null == (_item_attributes = item.attributes) ? void 0 : _item_attributes.nodeType) === NodeType.POSITION) && item.isVisible) matchingElements.push(item);
-            }
-        }
-        for (const child of node.children)dfs(child);
-    }
-    dfs(treeRoot);
-    if (0 === matchingElements.length) return;
-    const element = matchingElements.reduce((smallest, current)=>{
-        const smallestArea = smallest.rect.width * smallest.rect.height;
-        const currentArea = current.rect.width * current.rect.height;
-        return currentArea < smallestArea ? current : smallest;
-    });
-    const distanceToCenter = distance({
-        x: element.center[0],
-        y: element.center[1]
-    }, position);
-    if (requireStrictDistance) return distanceToCenter <= distanceThreshold ? element : void 0;
-    return element;
-}
 function distance(point1, point2) {
     return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);
 }
@@ -64,61 +31,9 @@ Description of all the elements in screenshot:
 </div>
 ====================
 `;
-async function describeUserPage(context, opt) {
-    const { screenshotBase64 } = context;
-    let width;
-    let height;
-    if (context.size) ({ width, height } = context.size);
-    else {
-        const imgSize = await imageInfoOfBase64(screenshotBase64);
-        ({ width, height } = imgSize);
-    }
-    const treeRoot = context.tree;
-    const idElementMap = {};
-    const flatElements = treeToList(treeRoot);
-    if ((null == opt ? void 0 : opt.domIncluded) === true && flatElements.length >= 5000) console.warn('The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: "visible-only" to reduce the number of elements');
-    flatElements.forEach((element)=>{
-        idElementMap[element.id] = element;
-        if (void 0 !== element.indexId) idElementMap[`${element.indexId}`] = element;
-    });
-    let pageDescription = '';
-    const visibleOnly = (null == opt ? void 0 : opt.domIncluded) === 'visible-only' ? true : (null == opt ? void 0 : opt.visibleOnly) ?? false;
-    const resolvedVlMode = null == opt ? void 0 : opt.vlMode;
-    const shouldIncludeDOM = (null == opt ? void 0 : opt.domIncluded) || !resolvedVlMode;
-    if (shouldIncludeDOM) {
-        const contentTree = await descriptionOfTree(treeRoot, null == opt ? void 0 : opt.truncateTextLength, null == opt ? void 0 : opt.filterNonTextContent, visibleOnly);
-        const sizeDescription = describeSize({
-            width,
-            height
-        });
-        pageDescription = `The size of the page: ${sizeDescription} \n The page elements tree:\n${contentTree}`;
-    }
-    return {
-        description: pageDescription,
-        elementById (idOrIndexId) {
-            assert(void 0 !== idOrIndexId, 'id is required for query');
-            const item = idElementMap[`${idOrIndexId}`];
-            return item;
-        },
-        elementByPosition (position, size) {
-            return elementByPositionWithElementInfo(treeRoot, position);
-        },
-        insertElementByPosition (position) {
-            const element = generateElementByPosition(position);
-            treeRoot.children.push({
-                node: element,
-                children: []
-            });
-            flatElements.push(element);
-            idElementMap[element.id] = element;
-            return element;
-        },
-        size: {
-            width,
-            height
-        }
-    };
+async function describeUserPage(context) {
+    return `The size of the page: ${describeSize(context.size)}`;
 }
-export { describeElement, describeSize, describeUserPage, distance, distanceThreshold, elementByPositionWithElementInfo, samplePageDescription };
+export { describeElement, describeSize, describeUserPage, distance, distanceThreshold, samplePageDescription };
 
 //# sourceMappingURL=util.mjs.map

package/dist/es/ai-model/prompt/util.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/util.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/util.ts"],"sourcesContent":["import { imageInfoOfBase64 } from '@/image/index';\nimport type { BaseElement, ElementTreeNode, Size, UIContext } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport {\n  descriptionOfTree,\n  generateElementByPosition,\n  treeToList,\n} from '@midscene/shared/extractor';\nimport { assert } from '@midscene/shared/utils';\n\nexport function describeSize(size: Size) {\n  return `${size.width} x ${size.height}`;\n}\n\nexport function describeElement(\n  elements: (Pick<BaseElement, 'rect' | 'content'> & { id: string })[],\n) {\n  const sliceLength = 80;\n  return elements\n    .map((item) =>\n      [\n        item.id,\n        item.rect.left,\n        item.rect.top,\n        item.rect.left + item.rect.width,\n        item.rect.top + item.rect.height,\n        item.content.length > sliceLength\n          ? `${item.content.slice(0, sliceLength)}...`\n          : item.content,\n      ].join(', '),\n    )\n    .join('\\n');\n}\nexport const distanceThreshold = 16;\n\nexport function elementByPositionWithElementInfo(\n  treeRoot: ElementTreeNode<BaseElement>,\n  position: {\n    x: number;\n    y: number;\n  },\n  options?: {\n    requireStrictDistance?: boolean;\n    filterPositionElements?: boolean;\n  },\n) {\n  const requireStrictDistance = options?.requireStrictDistance ?? true;\n  const filterPositionElements = options?.filterPositionElements ?? false;\n\n  assert(typeof position !== 'undefined', 'position is required for query');\n\n  const matchingElements: BaseElement[] = [];\n\n  function dfs(node: ElementTreeNode<BaseElement>) {\n    if (node?.node) {\n      const item = node.node;\n      if (\n        item.rect.left <= position.x &&\n        position.x <= item.rect.left + item.rect.width &&\n        item.rect.top <= position.y &&\n        position.y <= item.rect.top + item.rect.height\n      ) {\n        if (\n          !(\n            filterPositionElements &&\n            item.attributes?.nodeType === NodeType.POSITION\n          ) &&\n          item.isVisible\n        ) {\n          matchingElements.push(item);\n        }\n      }\n    }\n\n    for (const child of node.children) {\n      dfs(child);\n    }\n  }\n\n  dfs(treeRoot);\n\n  if (matchingElements.length === 0) {\n    return undefined;\n  }\n\n  // Find the smallest element by area\n  const element = matchingElements.reduce((smallest, current) => {\n    const smallestArea = smallest.rect.width * smallest.rect.height;\n    const currentArea = current.rect.width * current.rect.height;\n    return currentArea < smallestArea ? current : smallest;\n  });\n\n  const distanceToCenter = distance(\n    { x: element.center[0], y: element.center[1] },\n    position,\n  );\n\n  if (requireStrictDistance) {\n    return distanceToCenter <= distanceThreshold ? element : undefined;\n  }\n\n  return element;\n}\n\nexport function distance(\n  point1: { x: number; y: number },\n  point2: { x: number; y: number },\n) {\n  return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);\n}\n\nexport const samplePageDescription = `\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n`;\n\nexport async function describeUserPage<\n  ElementType extends BaseElement = BaseElement,\n>(\n  context: Omit<UIContext<ElementType>, 'describer'>,\n  opt: {\n    truncateTextLength?: number;\n    filterNonTextContent?: boolean;\n    domIncluded?: boolean | 'visible-only';\n    visibleOnly?: boolean;\n    vlMode: TVlModeTypes | undefined;\n  },\n) {\n  const { screenshotBase64 } = context;\n  let width: number;\n  let height: number;\n\n  if (context.size) {\n    ({ width, height } = context.size);\n  } else {\n    const imgSize = await imageInfoOfBase64(screenshotBase64);\n    ({ width, height } = imgSize);\n  }\n\n  const treeRoot = context.tree;\n  // dfs tree, save the id and element info\n  const idElementMap: Record<string, ElementType> = {};\n  const flatElements: ElementType[] = treeToList(treeRoot);\n\n  if (opt?.domIncluded === true && flatElements.length >= 5000) {\n    console.warn(\n      'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: \"visible-only\" to reduce the number of elements',\n    );\n  }\n\n  flatElements.forEach((element) => {\n    idElementMap[element.id] = element;\n    if (typeof element.indexId !== 'undefined') {\n      idElementMap[`${element.indexId}`] = element;\n    }\n  });\n\n  let pageDescription = '';\n\n  const visibleOnly =\n    opt?.domIncluded === 'visible-only' ? true : (opt?.visibleOnly ?? false);\n  const resolvedVlMode = opt?.vlMode;\n  const shouldIncludeDOM = opt?.domIncluded || !resolvedVlMode;\n\n  if (shouldIncludeDOM) {\n    // non-vl mode must provide the page description\n    const contentTree = await descriptionOfTree(\n      treeRoot,\n      opt?.truncateTextLength,\n      opt?.filterNonTextContent,\n      visibleOnly,\n    );\n\n    // if match by position, don't need to provide the page description\n    const sizeDescription = describeSize({ width, height });\n    pageDescription = `The size of the page: ${sizeDescription} \\n The page elements tree:\\n${contentTree}`;\n  }\n\n  return {\n    description: pageDescription,\n    elementById(idOrIndexId: string) {\n      assert(typeof idOrIndexId !== 'undefined', 'id is required for query');\n      const item = idElementMap[`${idOrIndexId}`];\n      return item;\n    },\n    elementByPosition(\n      position: { x: number; y: number },\n      size: { width: number; height: number },\n    ) {\n      return elementByPositionWithElementInfo(treeRoot, position);\n    },\n    insertElementByPosition(position: { x: number; y: number }) {\n      const element = generateElementByPosition(position) as ElementType;\n\n      treeRoot.children.push({\n        node: element,\n        children: [],\n      });\n      flatElements.push(element);\n      idElementMap[element.id] = element;\n      return element;\n    },\n    size: { width, height },\n  };\n}\n"],"names":["describeSize","size","describeElement","elements","sliceLength","item","distanceThreshold","elementByPositionWithElementInfo","treeRoot","position","options","requireStrictDistance","filterPositionElements","assert","matchingElements","dfs","node","_item_attributes","NodeType","child","element","smallest","current","smallestArea","currentArea","distanceToCenter","distance","undefined","point1","point2","Math","samplePageDescription","describeUserPage","context","opt","screenshotBase64","width","height","imgSize","imageInfoOfBase64","idElementMap","flatElements","treeToList","console","pageDescription","visibleOnly","resolvedVlMode","shouldIncludeDOM","contentTree","descriptionOfTree","sizeDescription","idOrIndexId","generateElementByPosition"],"mappings":";;;;AAWO,SAASA,aAAaC,IAAU;IACrC,OAAO,GAAGA,KAAK,KAAK,CAAC,GAAG,EAAEA,KAAK,MAAM,EAAE;AACzC;AAEO,SAASC,gBACdC,QAAoE;IAEpE,MAAMC,cAAc;IACpB,OAAOD,SACJ,GAAG,CAAC,CAACE,OACJ;YACEA,KAAK,EAAE;YACPA,KAAK,IAAI,CAAC,IAAI;YACdA,KAAK,IAAI,CAAC,GAAG;YACbA,KAAK,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI,CAAC,KAAK;YAChCA,KAAK,IAAI,CAAC,GAAG,GAAGA,KAAK,IAAI,CAAC,MAAM;YAChCA,KAAK,OAAO,CAAC,MAAM,GAAGD,cAClB,GAAGC,KAAK,OAAO,CAAC,KAAK,CAAC,GAAGD,aAAa,GAAG,CAAC,GAC1CC,KAAK,OAAO;SACjB,CAAC,IAAI,CAAC,OAER,IAAI,CAAC;AACV;AACO,MAAMC,oBAAoB;AAE1B,SAASC,iCACdC,QAAsC,EACtCC,QAGC,EACDC,OAGC;IAED,MAAMC,wBAAwBD,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,qBAAqB,AAAD,KAAK;IAChE,MAAME,yBAAyBF,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,sBAAsB,AAAD,KAAK;IAElEG,OAAO,AAAoB,WAAbJ,UAA0B;IAExC,MAAMK,mBAAkC,EAAE;IAE1C,SAASC,IAAIC,IAAkC;QAC7C,IAAIA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,EAAE;YACd,MAAMX,OAAOW,KAAK,IAAI;YACtB,IACEX,KAAK,IAAI,CAAC,IAAI,IAAII,SAAS,CAAC,IAC5BA,SAAS,CAAC,IAAIJ,KAAK,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI,CAAC,KAAK,IAC9CA,KAAK,IAAI,CAAC,GAAG,IAAII,SAAS,CAAC,IAC3BA,SAAS,CAAC,IAAIJ,KAAK,IAAI,CAAC,GAAG,GAAGA,KAAK,IAAI,CAAC,MAAM,EAC9C;oBAIIY;gBAHJ,IACE,CACEL,CAAAA,0BACAK,AAAAA,SAAAA,CAAAA,mBAAAA,KAAK,UAAU,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,QAAQ,AAAD,MAAMC,SAAS,QAAO,KAEhDb,KAAK,SAAS,EAEdS,iBAAiB,IAAI,CAACT;YAE1B;QACF;QAEA,KAAK,MAAMc,SAASH,KAAK,QAAQ,CAC/BD,IAAII;IAER;IAEAJ,IAAIP;IAEJ,IAAIM,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB;IAIF,MAAMM,UAAUN,iBAAiB,MAAM,CAAC,CAACO,UAAUC;QACjD,MAAMC,eAAeF,SAAS,IAAI,CAAC,KAAK,GAAGA,SAAS,IAAI,CAAC,MAAM;QAC/D,MAAMG,cAAcF,QAAQ,IAAI,CAAC,KAAK,GAAGA,QAAQ,IAAI,CAAC,MAAM;QAC5D,OAAOE,cAAcD,eAAeD,UAAUD;IAChD;IAEA,MAAMI,mBAAmBC,SACvB;QAAE,GAAGN,QAAQ,MAAM,CAAC,EAAE;QAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;IAAC,GAC7CX;IAGF,IAAIE,uBACF,OAAOc,oBAAoBnB,oBAAoBc,UAAUO;IAG3D,OAAOP;AACT;AAEO,SAASM,SACdE,MAAgC,EAChCC,MAAgC;IAEhC,OAAOC,KAAK,IAAI,CAAEF,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM,IAAKD,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM;AACzE;AAEO,MAAME,wBAAwB,CAAC;;;;;;;;;;;;;;AActC,CAAC;AAEM,eAAeC,iBAGpBC,OAAkD,EAClDC,GAMC;IAED,MAAM,EAAEC,gBAAgB,EAAE,GAAGF;IAC7B,IAAIG;IACJ,IAAIC;IAEJ,IAAIJ,QAAQ,IAAI,EACb,GAAEG,KAAK,EAAEC,MAAM,EAAE,GAAGJ,QAAQ,IAAG;SAC3B;QACL,MAAMK,UAAU,MAAMC,kBAAkBJ;QACvC,GAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGC,OAAM;IAC7B;IAEA,MAAM9B,WAAWyB,QAAQ,IAAI;IAE7B,MAAMO,eAA4C,CAAC;IACnD,MAAMC,eAA8BC,WAAWlC;IAE/C,IAAI0B,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM,QAAQO,aAAa,MAAM,IAAI,MACtDE,QAAQ,IAAI,CACV;IAIJF,aAAa,OAAO,CAAC,CAACrB;QACpBoB,YAAY,CAACpB,QAAQ,EAAE,CAAC,GAAGA;QAC3B,IAAI,AAA2B,WAApBA,QAAQ,OAAO,EACxBoB,YAAY,CAAC,GAAGpB,QAAQ,OAAO,EAAE,CAAC,GAAGA;IAEzC;IAEA,IAAIwB,kBAAkB;IAEtB,MAAMC,cACJX,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM,iBAAiB,OAAQA,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK;IACpE,MAAMY,iBAAiBZ,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,MAAM;IAClC,MAAMa,mBAAmBb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK,CAACY;IAE9C,IAAIC,kBAAkB;QAEpB,MAAMC,cAAc,MAAMC,kBACxBzC,UACA0B,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,kBAAkB,EACvBA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,oBAAoB,EACzBW;QAIF,MAAMK,kBAAkBlD,aAAa;YAAEoC;YAAOC;QAAO;QACrDO,kBAAkB,CAAC,sBAAsB,EAAEM,gBAAgB,6BAA6B,EAAEF,aAAa;IACzG;IAEA,OAAO;QACL,aAAaJ;QACb,aAAYO,WAAmB;YAC7BtC,OAAO,AAAuB,WAAhBsC,aAA6B;YAC3C,MAAM9C,OAAOmC,YAAY,CAAC,GAAGW,aAAa,CAAC;YAC3C,OAAO9C;QACT;QACA,mBACEI,QAAkC,EAClCR,IAAuC;YAEvC,OAAOM,iCAAiCC,UAAUC;QACpD;QACA,yBAAwBA,QAAkC;YACxD,MAAMW,UAAUgC,0BAA0B3C;YAE1CD,SAAS,QAAQ,CAAC,IAAI,CAAC;gBACrB,MAAMY;gBACN,UAAU,EAAE;YACd;YACAqB,aAAa,IAAI,CAACrB;YAClBoB,YAAY,CAACpB,QAAQ,EAAE,CAAC,GAAGA;YAC3B,OAAOA;QACT;QACA,MAAM;YAAEgB;YAAOC;QAAO;IACxB;AACF"}
+{"version":3,"file":"ai-model/prompt/util.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/util.ts"],"sourcesContent":["import type { BaseElement, ElementTreeNode, Size, UIContext } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport { assert } from '@midscene/shared/utils';\n\nexport function describeSize(size: Size) {\n  return `${size.width} x ${size.height}`;\n}\n\nexport function describeElement(\n  elements: (Pick<BaseElement, 'rect' | 'content'> & { id: string })[],\n) {\n  const sliceLength = 80;\n  return elements\n    .map((item) =>\n      [\n        item.id,\n        item.rect.left,\n        item.rect.top,\n        item.rect.left + item.rect.width,\n        item.rect.top + item.rect.height,\n        item.content.length > sliceLength\n          ? `${item.content.slice(0, sliceLength)}...`\n          : item.content,\n      ].join(', '),\n    )\n    .join('\\n');\n}\nexport const distanceThreshold = 16;\n\n// export function elementByPositionWithElementInfo(\n//   treeRoot: ElementTreeNode<BaseElement>,\n//   position: {\n//     x: number;\n//     y: number;\n//   },\n//   options?: {\n//     requireStrictDistance?: boolean;\n//     filterPositionElements?: boolean;\n//   },\n// ) {\n//   const requireStrictDistance = options?.requireStrictDistance ?? true;\n//   const filterPositionElements = options?.filterPositionElements ?? false;\n\n//   assert(typeof position !== 'undefined', 'position is required for query');\n\n//   const matchingElements: BaseElement[] = [];\n\n//   function dfs(node: ElementTreeNode<BaseElement>) {\n//     if (node?.node) {\n//       const item = node.node;\n//       if (\n//         item.rect.left <= position.x &&\n//         position.x <= item.rect.left + item.rect.width &&\n//         item.rect.top <= position.y &&\n//         position.y <= item.rect.top + item.rect.height\n//       ) {\n//         if (\n//           !(\n//             filterPositionElements &&\n//             item.attributes?.nodeType === NodeType.POSITION\n//           ) &&\n//           item.isVisible\n//         ) {\n//           matchingElements.push(item);\n//         }\n//       }\n//     }\n\n//     for (const child of node.children) {\n//       dfs(child);\n//     }\n//   }\n\n//   dfs(treeRoot);\n\n//   if (matchingElements.length === 0) {\n//     return undefined;\n//   }\n\n//   // Find the smallest element by area\n//   const element = matchingElements.reduce((smallest, current) => {\n//     const smallestArea = smallest.rect.width * smallest.rect.height;\n//     const currentArea = current.rect.width * current.rect.height;\n//     return currentArea < smallestArea ? current : smallest;\n//   });\n\n//   const distanceToCenter = distance(\n//     { x: element.center[0], y: element.center[1] },\n//     position,\n//   );\n\n//   if (requireStrictDistance) {\n//     return distanceToCenter <= distanceThreshold ? element : undefined;\n//   }\n\n//   return element;\n// }\n\nexport function distance(\n  point1: { x: number; y: number },\n  point2: { x: number; y: number },\n) {\n  return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);\n}\n\nexport const samplePageDescription = `\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n`;\n\nexport async function describeUserPage(context: UIContext) {\n  return `The size of the page: ${describeSize(context.size)}`;\n}\n"],"names":["describeSize","size","describeElement","elements","sliceLength","item","distanceThreshold","distance","point1","point2","Math","samplePageDescription","describeUserPage","context"],"mappings":"AAIO,SAASA,aAAaC,IAAU;IACrC,OAAO,GAAGA,KAAK,KAAK,CAAC,GAAG,EAAEA,KAAK,MAAM,EAAE;AACzC;AAEO,SAASC,gBACdC,QAAoE;IAEpE,MAAMC,cAAc;IACpB,OAAOD,SACJ,GAAG,CAAC,CAACE,OACJ;YACEA,KAAK,EAAE;YACPA,KAAK,IAAI,CAAC,IAAI;YACdA,KAAK,IAAI,CAAC,GAAG;YACbA,KAAK,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI,CAAC,KAAK;YAChCA,KAAK,IAAI,CAAC,GAAG,GAAGA,KAAK,IAAI,CAAC,MAAM;YAChCA,KAAK,OAAO,CAAC,MAAM,GAAGD,cAClB,GAAGC,KAAK,OAAO,CAAC,KAAK,CAAC,GAAGD,aAAa,GAAG,CAAC,GAC1CC,KAAK,OAAO;SACjB,CAAC,IAAI,CAAC,OAER,IAAI,CAAC;AACV;AACO,MAAMC,oBAAoB;AAuE1B,SAASC,SACdC,MAAgC,EAChCC,MAAgC;IAEhC,OAAOC,KAAK,IAAI,CAAEF,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM,IAAKD,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM;AACzE;AAEO,MAAME,wBAAwB,CAAC;;;;;;;;;;;;;;AActC,CAAC;AAEM,eAAeC,iBAAiBC,OAAkB;IACvD,OAAO,CAAC,sBAAsB,EAAEb,aAAaa,QAAQ,IAAI,GAAG;AAC9D"}

package/dist/es/ai-model/service-caller/index.mjs

@@ -11,37 +11,29 @@ import { assertSchema } from "../prompt/assertion.mjs";
 import { planSchema } from "../prompt/llm-planning.mjs";
 async function createChatClient({ AIActionTypeValue, modelConfig }) {
     const { socksProxy, httpProxy, modelName, openaiBaseURL, openaiApiKey, openaiExtraConfig, modelDescription, uiTarsModelVersion: uiTarsVersion, vlMode, createOpenAIClient } = modelConfig;
-    let openai;
-    if (createOpenAIClient) openai = createOpenAIClient({
-        modelName,
-        openaiApiKey,
-        openaiBaseURL,
-        socksProxy,
-        httpProxy,
-        openaiExtraConfig,
-        vlMode,
-        intent: modelConfig.intent,
-        modelDescription
-    });
-    else {
-        let proxyAgent;
-        const debugProxy = getDebug('ai:call:proxy');
-        if (httpProxy) {
-            debugProxy('using http proxy', httpProxy);
-            proxyAgent = new HttpsProxyAgent(httpProxy);
-        } else if (socksProxy) {
-            debugProxy('using socks proxy', socksProxy);
-            proxyAgent = new SocksProxyAgent(socksProxy);
-        }
-        openai = new openai_0({
-            baseURL: openaiBaseURL,
-            apiKey: openaiApiKey,
-            ...proxyAgent ? {
-                httpAgent: proxyAgent
-            } : {},
-            ...openaiExtraConfig,
-            dangerouslyAllowBrowser: true
-        });
+    let proxyAgent;
+    const debugProxy = getDebug('ai:call:proxy');
+    if (httpProxy) {
+        debugProxy('using http proxy', httpProxy);
+        proxyAgent = new HttpsProxyAgent(httpProxy);
+    } else if (socksProxy) {
+        debugProxy('using socks proxy', socksProxy);
+        proxyAgent = new SocksProxyAgent(socksProxy);
+    }
+    const openAIOptions = {
+        baseURL: openaiBaseURL,
+        apiKey: openaiApiKey,
+        ...proxyAgent ? {
+            httpAgent: proxyAgent
+        } : {},
+        ...openaiExtraConfig,
+        dangerouslyAllowBrowser: true
+    };
+    const baseOpenAI = new openai_0(openAIOptions);
+    let openai = baseOpenAI;
+    if (createOpenAIClient) {
+        const wrappedClient = await createOpenAIClient(baseOpenAI, openAIOptions);
+        if (wrappedClient) openai = wrappedClient;
     }
     return {
         completion: openai.chat.completions,

package/dist/es/ai-model/service-caller/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport {\n  type IModelConfig,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelConfig,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n    createOpenAIClient,\n  } = modelConfig;\n\n  let openai: OpenAI;\n\n  // Use custom client factory if provided\n  if (createOpenAIClient) {\n    openai = createOpenAIClient({\n      modelName,\n      openaiApiKey,\n      openaiBaseURL,\n      socksProxy,\n      httpProxy,\n      openaiExtraConfig,\n      vlMode,\n      intent: modelConfig.intent,\n      modelDescription,\n    });\n  } else {\n    // Default OpenAI client creation\n    let proxyAgent = undefined;\n    const debugProxy = getDebug('ai:call:proxy');\n    if (httpProxy) {\n      debugProxy('using http proxy', httpProxy);\n      proxyAgent = new HttpsProxyAgent(httpProxy);\n    } else if (socksProxy) {\n      debugProxy('using socks proxy', socksProxy);\n      proxyAgent = new SocksProxyAgent(socksProxy);\n    }\n\n    openai = new OpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      ...(proxyAgent ? { httpAgent: proxyAgent as any } : {}),\n      ...openaiExtraConfig,\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  return {\n    completion: openai.chat.completions,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  };\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const { completion, modelName, modelDescription, uiTarsVersion, vlMode } =\n    await createChatClient({\n      AIActionTypeValue,\n      modelConfig,\n    });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens = globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlMode === 'vlm-ui-tars' ? 0.0 : 0.1,\n    stream: !!isStreaming,\n    max_tokens:\n      typeof maxTokens === 'number'\n        ? maxTokens\n        : Number.parseInt(maxTokens || '2048', 10),\n    ...(vlMode === 'qwen-vl' // qwen vl v2 specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    debugCall(\n      `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n    );\n\n    if (isStreaming) {\n      const stream = (await completion.create(\n        {\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        },\n        {\n          stream: true,\n        },\n      )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n        _request_id?: string | null;\n      };\n\n      for await (const chunk of stream) {\n        const content = chunk.choices?.[0]?.delta?.content || '';\n        const reasoning_content =\n          (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n        // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n        if (chunk.usage) {\n          usage = chunk.usage;\n        }\n\n        if (content || reasoning_content) {\n          accumulated += content;\n          const chunkData: CodeGenerationChunk = {\n            content,\n            reasoning_content,\n            accumulated,\n            isComplete: false,\n            usage: undefined,\n          };\n          options.onChunk!(chunkData);\n        }\n\n        // Check if stream is complete\n        if (chunk.choices?.[0]?.finish_reason) {\n          timeCost = Date.now() - startTime;\n\n          // If usage is not available from the stream, provide a basic usage info\n          if (!usage) {\n            // Estimate token counts based on content length (rough approximation)\n            const estimatedTokens = Math.max(\n              1,\n              Math.floor(accumulated.length / 4),\n            );\n            usage = {\n              prompt_tokens: estimatedTokens,\n              completion_tokens: estimatedTokens,\n              total_tokens: estimatedTokens * 2,\n            };\n          }\n\n          // Send final chunk\n          const finalChunk: CodeGenerationChunk = {\n            content: '',\n            accumulated,\n            reasoning_content: '',\n            isComplete: true,\n            usage: {\n              prompt_tokens: usage.prompt_tokens ?? 0,\n              completion_tokens: usage.completion_tokens ?? 0,\n              total_tokens: usage.total_tokens ?? 0,\n              time_cost: timeCost ?? 0,\n              model_name: modelName,\n              model_description: modelDescription,\n              intent: modelConfig.intent,\n            },\n          };\n          options.onChunk!(finalChunk);\n          break;\n        }\n      }\n      content = accumulated;\n      debugProfileStats(\n        `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`,\n      );\n    } else {\n      const result = await completion.create({\n        model: modelName,\n        messages,\n        response_format: responseFormat,\n        ...commonConfig,\n      } as any);\n      timeCost = Date.now() - startTime;\n\n      debugProfileStats(\n        `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n      );\n\n      debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);\n\n      assert(\n        result.choices,\n        `invalid response from LLM service: ${JSON.stringify(result)}`,\n      );\n      content = result.choices[0].message.content!;\n      usage = result.usage;\n    }\n\n    debugCall(`response: ${content}`);\n    assert(content, 'empty content');\n\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: modelName,\n            model_description: modelDescription,\n            intent: modelConfig.intent,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n      case AIActionType.TEXT:\n        // No response format for plain text - return as-is\n        responseFormat = undefined;\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  // Skip for plain text to allow string output\n  if (\n    modelName === 'gpt-4o-2024-05-13' &&\n    AIActionTypeValue !== AIActionType.TEXT\n  ) {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callAIWithObjectResponse<T>(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const response = await callAI(messages, AIActionTypeValue, modelConfig);\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  return { content: jsonContent, usage: response.usage };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n  try {\n    return JSON.parse(cleanJsonString);\n  } catch {}\n  try {\n    return JSON.parse(jsonrepair(cleanJsonString));\n  } catch (e) {}\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    return JSON.parse(jsonrepair(jsonString));\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["createChatClient","AIActionTypeValue","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","modelDescription","uiTarsVersion","vlMode","createOpenAIClient","openai","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","OpenAI","callAI","messages","options","completion","responseFormat","getResponseFormat","maxTokens","globalConfigManager","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","JSON","assert","e","console","newError","Error","AIActionType","assertSchema","planSchema","AIResponseFormat","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;AAsBA,eAAeA,iBAAiB,EAC9BC,iBAAiB,EACjBC,WAAW,EAIZ;IAOC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACNC,kBAAkB,EACnB,GAAGV;IAEJ,IAAIW;IAGJ,IAAID,oBACFC,SAASD,mBAAmB;QAC1BP;QACAE;QACAD;QACAH;QACAC;QACAI;QACAG;QACA,QAAQT,YAAY,MAAM;QAC1BO;IACF;SACK;QAEL,IAAIK;QACJ,MAAMC,aAAaC,SAAS;QAC5B,IAAIZ,WAAW;YACbW,WAAW,oBAAoBX;YAC/BU,aAAa,IAAIG,gBAAgBb;QACnC,OAAO,IAAID,YAAY;YACrBY,WAAW,qBAAqBZ;YAChCW,aAAa,IAAII,gBAAgBf;QACnC;QAEAU,SAAS,IAAIM,SAAO;YAClB,SAASb;YACT,QAAQC;YACR,GAAIO,aAAa;gBAAE,WAAWA;YAAkB,IAAI,CAAC,CAAC;YACtD,GAAGN,iBAAiB;YACpB,yBAAyB;QAC3B;IACF;IAEA,OAAO;QACL,YAAYK,OAAO,IAAI,CAAC,WAAW;QACnCR;QACAI;QACAC;QACAC;IACF;AACF;AAEO,eAAeS,OACpBC,QAAsC,EACtCpB,iBAA+B,EAC/BC,WAAyB,EACzBoB,OAGC;IAED,MAAM,EAAEC,UAAU,EAAElB,SAAS,EAAEI,gBAAgB,EAAEC,aAAa,EAAEC,MAAM,EAAE,GACtE,MAAMX,iBAAiB;QACrBC;QACAC;IACF;IAEF,MAAMsB,iBAAiBC,kBAAkBpB,WAAWJ;IAEpD,MAAMyB,YAAYC,oBAAoB,iBAAiB,CAACC;IACxD,MAAMC,YAAYb,SAAS;IAC3B,MAAMc,oBAAoBd,SAAS;IACnC,MAAMe,qBAAqBf,SAAS;IAEpC,MAAMgB,YAAYC,KAAK,GAAG;IAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIa;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,eAAe;QACnB,aAAa5B,AAAW,kBAAXA,SAA2B,MAAM;QAC9C,QAAQ,CAAC,CAACuB;QACV,YACE,AAAqB,YAArB,OAAOR,YACHA,YACAc,OAAO,QAAQ,CAACd,aAAa,QAAQ;QAC3C,GAAIf,AAAW,cAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACFkB,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAE7B,WAAW;QAGrE,IAAI6B,aAAa;YACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CACrC;gBACE,OAAOlB;gBACPgB;gBACA,iBAAiBG;gBACjB,GAAGe,YAAY;YACjB,GACA;gBACE,QAAQ;YACV;YAKF,WAAW,MAAMG,SAASD,OAAQ;oBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;gBAtBJ,MAAMV,UAAUQ,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;gBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;gBAG3D,IAAIF,MAAM,KAAK,EACbL,QAAQK,MAAM,KAAK;gBAGrB,IAAIP,WAAWW,mBAAmB;oBAChCV,eAAeD;oBACf,MAAMY,YAAiC;wBACrCZ;wBACAW;wBACAV;wBACA,YAAY;wBACZ,OAAOY;oBACT;oBACA1B,QAAQ,OAAO,CAAEyB;gBACnB;gBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;oBACrCP,WAAWL,KAAK,GAAG,KAAKD;oBAGxB,IAAI,CAACK,OAAO;wBAEV,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACd,YAAY,MAAM,GAAG;wBAElCC,QAAQ;4BACN,eAAeY;4BACf,mBAAmBA;4BACnB,cAAcA,AAAkB,IAAlBA;wBAChB;oBACF;oBAGA,MAAME,aAAkC;wBACtC,SAAS;wBACTf;wBACA,mBAAmB;wBACnB,YAAY;wBACZ,OAAO;4BACL,eAAeC,MAAM,aAAa,IAAI;4BACtC,mBAAmBA,MAAM,iBAAiB,IAAI;4BAC9C,cAAcA,MAAM,YAAY,IAAI;4BACpC,WAAWC,YAAY;4BACvB,YAAYjC;4BACZ,mBAAmBI;4BACnB,QAAQP,YAAY,MAAM;wBAC5B;oBACF;oBACAoB,QAAQ,OAAO,CAAE6B;oBACjB;gBACF;YACF;YACAhB,UAAUC;YACVN,kBACE,CAAC,iBAAiB,EAAEzB,UAAU,QAAQ,EAAEM,UAAU,UAAU,WAAW,EAAE2B,UAAU;QAEvF,OAAO;gBAUqGc,eAAyDC,gBAAwDC;YAT3N,MAAMC,SAAS,MAAMhC,WAAW,MAAM,CAAC;gBACrC,OAAOlB;gBACPgB;gBACA,iBAAiBG;gBACjB,GAAGe,YAAY;YACjB;YACAD,WAAWL,KAAK,GAAG,KAAKD;YAExBF,kBACE,CAAC,OAAO,EAAEzB,UAAU,QAAQ,EAAEM,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAE0C,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEhB,SAAS,aAAa,EAAEiB,OAAO,WAAW,IAAI,IAAI;YAG3TxB,mBAAmB,CAAC,oBAAoB,EAAEyB,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;YAExEE,OACEF,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;YAEhEpB,UAAUoB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;YAC3ClB,QAAQkB,OAAO,KAAK;QACtB;QAEA1B,UAAU,CAAC,UAAU,EAAEM,SAAS;QAChCsB,OAAOtB,SAAS;QAGhB,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEf,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAeY;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASd,WAAW;YACpB,OAAOE,QACH;gBACE,eAAeA,MAAM,aAAa,IAAI;gBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gBAC9C,cAAcA,MAAM,YAAY,IAAI;gBACpC,WAAWC,YAAY;gBACvB,YAAYjC;gBACZ,mBAAmBI;gBACnB,QAAQP,YAAY,MAAM;YAC5B,IACA8C;YACJ,YAAY,CAAC,CAACd;QAChB;IACF,EAAE,OAAOwB,GAAQ;QACfC,QAAQ,KAAK,CAAC,kBAAkBD;QAChC,MAAME,WAAW,IAAIC,MACnB,CAAC,eAAe,EAAE3B,cAAc,eAAe,GAAG,kBAAkB,EAAEwB,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAME;IACR;AACF;AAEO,MAAMnC,oBAAoB,CAC/BpB,WACAJ;IAIA,IAAIuB;IAKJ,IAAInB,UAAU,QAAQ,CAAC,UACrB,OAAQJ;QACN,KAAK6D,aAAa,MAAM;YACtBtC,iBAAiBuC;YACjB;QACF,KAAKD,aAAa,IAAI;YACpBtC,iBAAiBwC;YACjB;QACF,KAAKF,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChCtC,iBAAiB;gBAAE,MAAMyC,iBAAiB,IAAI;YAAC;YAC/C;QACF,KAAKH,aAAa,IAAI;YAEpBtC,iBAAiBwB;YACjB;IACJ;IAKF,IACE3C,AAAc,wBAAdA,aACAJ,sBAAsB6D,aAAa,IAAI,EAEvCtC,iBAAiB;QAAE,MAAMyC,iBAAiB,IAAI;IAAC;IAGjD,OAAOzC;AACT;AAEO,eAAe0C,yBACpB7C,QAAsC,EACtCpB,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAMiE,WAAW,MAAM/C,OAAOC,UAAUpB,mBAAmBC;IAC3DuD,OAAOU,UAAU;IACjB,MAAMxD,SAAST,YAAY,MAAM;IACjC,MAAMkE,cAAcC,cAAcF,SAAS,OAAO,EAAExD;IACpD,OAAO;QAAE,SAASyD;QAAa,OAAOD,SAAS,KAAK;IAAC;AACvD;AAEO,eAAeG,yBACpBC,IAAY,EACZtE,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM,EAAEiC,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,OAAOmD,MAAMtE,mBAAmBC;IACjE,OAAO;QAAEiC;QAASE;IAAM;AAC1B;AAEO,SAASmC,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASR,cAAcQ,KAAa,EAAElE,MAAgC;IAC3E,MAAMmE,kBAAkBN,yBAAyBK;IAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAACvC;IACT;IACA,IAAI;QACF,OAAOgB,KAAK,KAAK,CAACsB;IACpB,EAAE,OAAM,CAAC;IACT,IAAI;QACF,OAAOtB,KAAK,KAAK,CAACwB,WAAWF;IAC/B,EAAE,OAAOpB,GAAG,CAAC;IAEb,IAAI/C,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAMsE,aAAaL,yBAAyBE;QAC5C,OAAOtB,KAAK,KAAK,CAACwB,WAAWC;IAC/B;IACA,MAAMpB,MAAM,CAAC,+BAA+B,EAAEgB,OAAO;AACvD"}
+{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport {\n  type IModelConfig,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelConfig,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n    createOpenAIClient,\n  } = modelConfig;\n\n  let proxyAgent = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n  if (httpProxy) {\n    debugProxy('using http proxy', httpProxy);\n    proxyAgent = new HttpsProxyAgent(httpProxy);\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', socksProxy);\n    proxyAgent = new SocksProxyAgent(socksProxy);\n  }\n\n  const openAIOptions = {\n    baseURL: openaiBaseURL,\n    apiKey: openaiApiKey,\n    ...(proxyAgent ? { httpAgent: proxyAgent as any } : {}),\n    ...openaiExtraConfig,\n    dangerouslyAllowBrowser: true,\n  };\n\n  const baseOpenAI = new OpenAI(openAIOptions);\n\n  let openai: OpenAI = baseOpenAI;\n\n  if (createOpenAIClient) {\n    const wrappedClient = await createOpenAIClient(baseOpenAI, openAIOptions);\n\n    if (wrappedClient) {\n      openai = wrappedClient as OpenAI;\n    }\n  }\n\n  return {\n    completion: openai.chat.completions,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  };\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const { completion, modelName, modelDescription, uiTarsVersion, vlMode } =\n    await createChatClient({\n      AIActionTypeValue,\n      modelConfig,\n    });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens = globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlMode === 'vlm-ui-tars' ? 0.0 : 0.1,\n    stream: !!isStreaming,\n    max_tokens:\n      typeof maxTokens === 'number'\n        ? maxTokens\n        : Number.parseInt(maxTokens || '2048', 10),\n    ...(vlMode === 'qwen-vl' // qwen vl v2 specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    debugCall(\n      `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n    );\n\n    if (isStreaming) {\n      const stream = (await completion.create(\n        {\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        },\n        {\n          stream: true,\n        },\n      )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n        _request_id?: string | null;\n      };\n\n      for await (const chunk of stream) {\n        const content = chunk.choices?.[0]?.delta?.content || '';\n        const reasoning_content =\n          (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n        // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n        if (chunk.usage) {\n          usage = chunk.usage;\n        }\n\n        if (content || reasoning_content) {\n          accumulated += content;\n          const chunkData: CodeGenerationChunk = {\n            content,\n            reasoning_content,\n            accumulated,\n            isComplete: false,\n            usage: undefined,\n          };\n          options.onChunk!(chunkData);\n        }\n\n        // Check if stream is complete\n        if (chunk.choices?.[0]?.finish_reason) {\n          timeCost = Date.now() - startTime;\n\n          // If usage is not available from the stream, provide a basic usage info\n          if (!usage) {\n            // Estimate token counts based on content length (rough approximation)\n            const estimatedTokens = Math.max(\n              1,\n              Math.floor(accumulated.length / 4),\n            );\n            usage = {\n              prompt_tokens: estimatedTokens,\n              completion_tokens: estimatedTokens,\n              total_tokens: estimatedTokens * 2,\n            };\n          }\n\n          // Send final chunk\n          const finalChunk: CodeGenerationChunk = {\n            content: '',\n            accumulated,\n            reasoning_content: '',\n            isComplete: true,\n            usage: {\n              prompt_tokens: usage.prompt_tokens ?? 0,\n              completion_tokens: usage.completion_tokens ?? 0,\n              total_tokens: usage.total_tokens ?? 0,\n              time_cost: timeCost ?? 0,\n              model_name: modelName,\n              model_description: modelDescription,\n              intent: modelConfig.intent,\n            },\n          };\n          options.onChunk!(finalChunk);\n          break;\n        }\n      }\n      content = accumulated;\n      debugProfileStats(\n        `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`,\n      );\n    } else {\n      const result = await completion.create({\n        model: modelName,\n        messages,\n        response_format: responseFormat,\n        ...commonConfig,\n      } as any);\n      timeCost = Date.now() - startTime;\n\n      debugProfileStats(\n        `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n      );\n\n      debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);\n\n      assert(\n        result.choices,\n        `invalid response from LLM service: ${JSON.stringify(result)}`,\n      );\n      content = result.choices[0].message.content!;\n      usage = result.usage;\n    }\n\n    debugCall(`response: ${content}`);\n    assert(content, 'empty content');\n\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: modelName,\n            model_description: modelDescription,\n            intent: modelConfig.intent,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n      case AIActionType.TEXT:\n        // No response format for plain text - return as-is\n        responseFormat = undefined;\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  // Skip for plain text to allow string output\n  if (\n    modelName === 'gpt-4o-2024-05-13' &&\n    AIActionTypeValue !== AIActionType.TEXT\n  ) {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callAIWithObjectResponse<T>(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const response = await callAI(messages, AIActionTypeValue, modelConfig);\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  return { content: jsonContent, usage: response.usage };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n  try {\n    return JSON.parse(cleanJsonString);\n  } catch {}\n  try {\n    return JSON.parse(jsonrepair(cleanJsonString));\n  } catch (e) {}\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    return JSON.parse(jsonrepair(jsonString));\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["createChatClient","AIActionTypeValue","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","modelDescription","uiTarsVersion","vlMode","createOpenAIClient","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","openAIOptions","baseOpenAI","OpenAI","openai","wrappedClient","callAI","messages","options","completion","responseFormat","getResponseFormat","maxTokens","globalConfigManager","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","JSON","assert","e","console","newError","Error","AIActionType","assertSchema","planSchema","AIResponseFormat","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;AAsBA,eAAeA,iBAAiB,EAC9BC,iBAAiB,EACjBC,WAAW,EAIZ;IAOC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACNC,kBAAkB,EACnB,GAAGV;IAEJ,IAAIW;IACJ,MAAMC,aAAaC,SAAS;IAC5B,IAAIX,WAAW;QACbU,WAAW,oBAAoBV;QAC/BS,aAAa,IAAIG,gBAAgBZ;IACnC,OAAO,IAAID,YAAY;QACrBW,WAAW,qBAAqBX;QAChCU,aAAa,IAAII,gBAAgBd;IACnC;IAEA,MAAMe,gBAAgB;QACpB,SAASZ;QACT,QAAQC;QACR,GAAIM,aAAa;YAAE,WAAWA;QAAkB,IAAI,CAAC,CAAC;QACtD,GAAGL,iBAAiB;QACpB,yBAAyB;IAC3B;IAEA,MAAMW,aAAa,IAAIC,SAAOF;IAE9B,IAAIG,SAAiBF;IAErB,IAAIP,oBAAoB;QACtB,MAAMU,gBAAgB,MAAMV,mBAAmBO,YAAYD;QAE3D,IAAII,eACFD,SAASC;IAEb;IAEA,OAAO;QACL,YAAYD,OAAO,IAAI,CAAC,WAAW;QACnChB;QACAI;QACAC;QACAC;IACF;AACF;AAEO,eAAeY,OACpBC,QAAsC,EACtCvB,iBAA+B,EAC/BC,WAAyB,EACzBuB,OAGC;IAED,MAAM,EAAEC,UAAU,EAAErB,SAAS,EAAEI,gBAAgB,EAAEC,aAAa,EAAEC,MAAM,EAAE,GACtE,MAAMX,iBAAiB;QACrBC;QACAC;IACF;IAEF,MAAMyB,iBAAiBC,kBAAkBvB,WAAWJ;IAEpD,MAAM4B,YAAYC,oBAAoB,iBAAiB,CAACC;IACxD,MAAMC,YAAYjB,SAAS;IAC3B,MAAMkB,oBAAoBlB,SAAS;IACnC,MAAMmB,qBAAqBnB,SAAS;IAEpC,MAAMoB,YAAYC,KAAK,GAAG;IAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIa;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,eAAe;QACnB,aAAa/B,AAAW,kBAAXA,SAA2B,MAAM;QAC9C,QAAQ,CAAC,CAAC0B;QACV,YACE,AAAqB,YAArB,OAAOR,YACHA,YACAc,OAAO,QAAQ,CAACd,aAAa,QAAQ;QAC3C,GAAIlB,AAAW,cAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACFqB,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAEhC,WAAW;QAGrE,IAAIgC,aAAa;YACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CACrC;gBACE,OAAOrB;gBACPmB;gBACA,iBAAiBG;gBACjB,GAAGe,YAAY;YACjB,GACA;gBACE,QAAQ;YACV;YAKF,WAAW,MAAMG,SAASD,OAAQ;oBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;gBAtBJ,MAAMV,UAAUQ,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;gBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;gBAG3D,IAAIF,MAAM,KAAK,EACbL,QAAQK,MAAM,KAAK;gBAGrB,IAAIP,WAAWW,mBAAmB;oBAChCV,eAAeD;oBACf,MAAMY,YAAiC;wBACrCZ;wBACAW;wBACAV;wBACA,YAAY;wBACZ,OAAOY;oBACT;oBACA1B,QAAQ,OAAO,CAAEyB;gBACnB;gBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;oBACrCP,WAAWL,KAAK,GAAG,KAAKD;oBAGxB,IAAI,CAACK,OAAO;wBAEV,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACd,YAAY,MAAM,GAAG;wBAElCC,QAAQ;4BACN,eAAeY;4BACf,mBAAmBA;4BACnB,cAAcA,AAAkB,IAAlBA;wBAChB;oBACF;oBAGA,MAAME,aAAkC;wBACtC,SAAS;wBACTf;wBACA,mBAAmB;wBACnB,YAAY;wBACZ,OAAO;4BACL,eAAeC,MAAM,aAAa,IAAI;4BACtC,mBAAmBA,MAAM,iBAAiB,IAAI;4BAC9C,cAAcA,MAAM,YAAY,IAAI;4BACpC,WAAWC,YAAY;4BACvB,YAAYpC;4BACZ,mBAAmBI;4BACnB,QAAQP,YAAY,MAAM;wBAC5B;oBACF;oBACAuB,QAAQ,OAAO,CAAE6B;oBACjB;gBACF;YACF;YACAhB,UAAUC;YACVN,kBACE,CAAC,iBAAiB,EAAE5B,UAAU,QAAQ,EAAEM,UAAU,UAAU,WAAW,EAAE8B,UAAU;QAEvF,OAAO;gBAUqGc,eAAyDC,gBAAwDC;YAT3N,MAAMC,SAAS,MAAMhC,WAAW,MAAM,CAAC;gBACrC,OAAOrB;gBACPmB;gBACA,iBAAiBG;gBACjB,GAAGe,YAAY;YACjB;YACAD,WAAWL,KAAK,GAAG,KAAKD;YAExBF,kBACE,CAAC,OAAO,EAAE5B,UAAU,QAAQ,EAAEM,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAE6C,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEhB,SAAS,aAAa,EAAEiB,OAAO,WAAW,IAAI,IAAI;YAG3TxB,mBAAmB,CAAC,oBAAoB,EAAEyB,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;YAExEE,OACEF,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;YAEhEpB,UAAUoB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;YAC3ClB,QAAQkB,OAAO,KAAK;QACtB;QAEA1B,UAAU,CAAC,UAAU,EAAEM,SAAS;QAChCsB,OAAOtB,SAAS;QAGhB,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEf,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAeY;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASd,WAAW;YACpB,OAAOE,QACH;gBACE,eAAeA,MAAM,aAAa,IAAI;gBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gBAC9C,cAAcA,MAAM,YAAY,IAAI;gBACpC,WAAWC,YAAY;gBACvB,YAAYpC;gBACZ,mBAAmBI;gBACnB,QAAQP,YAAY,MAAM;YAC5B,IACAiD;YACJ,YAAY,CAAC,CAACd;QAChB;IACF,EAAE,OAAOwB,GAAQ;QACfC,QAAQ,KAAK,CAAC,kBAAkBD;QAChC,MAAME,WAAW,IAAIC,MACnB,CAAC,eAAe,EAAE3B,cAAc,eAAe,GAAG,kBAAkB,EAAEwB,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAME;IACR;AACF;AAEO,MAAMnC,oBAAoB,CAC/BvB,WACAJ;IAIA,IAAI0B;IAKJ,IAAItB,UAAU,QAAQ,CAAC,UACrB,OAAQJ;QACN,KAAKgE,aAAa,MAAM;YACtBtC,iBAAiBuC;YACjB;QACF,KAAKD,aAAa,IAAI;YACpBtC,iBAAiBwC;YACjB;QACF,KAAKF,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChCtC,iBAAiB;gBAAE,MAAMyC,iBAAiB,IAAI;YAAC;YAC/C;QACF,KAAKH,aAAa,IAAI;YAEpBtC,iBAAiBwB;YACjB;IACJ;IAKF,IACE9C,AAAc,wBAAdA,aACAJ,sBAAsBgE,aAAa,IAAI,EAEvCtC,iBAAiB;QAAE,MAAMyC,iBAAiB,IAAI;IAAC;IAGjD,OAAOzC;AACT;AAEO,eAAe0C,yBACpB7C,QAAsC,EACtCvB,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAMoE,WAAW,MAAM/C,OAAOC,UAAUvB,mBAAmBC;IAC3D0D,OAAOU,UAAU;IACjB,MAAM3D,SAAST,YAAY,MAAM;IACjC,MAAMqE,cAAcC,cAAcF,SAAS,OAAO,EAAE3D;IACpD,OAAO;QAAE,SAAS4D;QAAa,OAAOD,SAAS,KAAK;IAAC;AACvD;AAEO,eAAeG,yBACpBC,IAAY,EACZzE,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM,EAAEoC,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,OAAOmD,MAAMzE,mBAAmBC;IACjE,OAAO;QAAEoC;QAASE;IAAM;AAC1B;AAEO,SAASmC,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASR,cAAcQ,KAAa,EAAErE,MAAgC;IAC3E,MAAMsE,kBAAkBN,yBAAyBK;IAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAACvC;IACT;IACA,IAAI;QACF,OAAOgB,KAAK,KAAK,CAACsB;IACpB,EAAE,OAAM,CAAC;IACT,IAAI;QACF,OAAOtB,KAAK,KAAK,CAACwB,WAAWF;IAC/B,EAAE,OAAOpB,GAAG,CAAC;IAEb,IAAIlD,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAMyE,aAAaL,yBAAyBE;QAC5C,OAAOtB,KAAK,KAAK,CAACwB,WAAWC;IAC/B;IACA,MAAMpB,MAAM,CAAC,+BAA+B,EAAEgB,OAAO;AACvD"}

package/dist/es/insight/index.mjs

@@ -51,7 +51,7 @@ class Insight {
             searchArea = searchAreaResponse.rect;
         }
         const startTime = Date.now();
-        const { parseResult, rect, elementById, rawResponse, usage, isOrderSensitive } = await AiLocateElement({
+        const { parseResult, rect, rawResponse, usage } = await AiLocateElement({
             callAIFn: this.aiVendorFn,
             context,
             targetElementDescription: queryPrompt,
@@ -83,14 +83,7 @@ class Insight {
             deepThink: !!searchArea,
             error: errorLog
         };
-        const elements = [];
-        (parseResult.elements || []).forEach((item)=>{
-            if ('id' in item) {
-                const element = elementById(null == item ? void 0 : item.id);
-                if (!element) return void console.warn(`locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`);
-                elements.push(element);
-            }
-        });
+        const elements = parseResult.elements || [];
         const dump = createInsightDump({
             ...dumpData,
             matchedElement: elements
@@ -100,12 +93,9 @@ class Insight {
         if (1 === elements.length) return {
             element: {
                 id: elements[0].id,
-                indexId: elements[0].indexId,
                 center: elements[0].center,
                 rect: elements[0].rect,
-                xpaths: elements[0].xpaths || [],
-                attributes: elements[0].attributes,
-                isOrderSensitive
+                isOrderSensitive: elements[0].isOrderSensitive
             },
             rect,
             dump
@@ -116,7 +106,7 @@ class Insight {
             dump
         };
     }
-    async extract(dataDemand, modelConfig, opt, multimodalPrompt) {
+    async extract(dataDemand, modelConfig, opt, pageDescription, multimodalPrompt) {
         var _parseResult_errors;
         assert('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
         const context = await this.contextRetrieverFn();
@@ -126,7 +116,8 @@ class Insight {
             dataQuery: dataDemand,
             multimodalPrompt,
             extractOption: opt,
-            modelConfig
+            modelConfig,
+            pageDescription
         });
         const timeCost = Date.now() - startTime;
         const taskInfo = {