@midscene/core 1.2.1-beta-20260114072539.0 → 1.2.1

package/dist/es/ai-model/inspect.mjs

@@ -3,14 +3,11 @@ import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl } from "@mi
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { adaptBboxToRect, expandSearchArea, mergeRects } from "../common.mjs";
-import { parseAutoGLMLocateResponse } from "./auto-glm/parser.mjs";
-import { getAutoGLMLocatePrompt } from "./auto-glm/prompt.mjs";
-import { isAutoGLM } from "./auto-glm/util.mjs";
 import { extractDataQueryPrompt, systemPromptToExtract } from "./prompt/extraction.mjs";
 import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
 import { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive } from "./prompt/order-sensitive-judge.mjs";
-import { callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
+import { callAIWithObjectResponse } from "./service-caller/index.mjs";
 const debugInspect = getDebug('ai:inspect');
 const debugSection = getDebug('ai:section');
 const extraTextFromUserPrompt = (prompt)=>{
@@ -63,7 +60,7 @@ async function AiLocateElement(options) {
     assert(targetElementDescription, "cannot find the target element description");
     const targetElementDescriptionText = extraTextFromUserPrompt(targetElementDescription);
     const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);
-    const systemPrompt = isAutoGLM(vlMode) ? getAutoGLMLocatePrompt(vlMode) : systemPromptToLocateElement(vlMode);
+    const systemPrompt = systemPromptToLocateElement(vlMode);
     let imagePayload = screenshotBase64;
     let imageWidth = context.size.width;
     let imageHeight = context.size.height;
@@ -100,7 +97,7 @@ async function AiLocateElement(options) {
                 },
                 {
                     type: 'text',
-                    text: isAutoGLM(vlMode) ? `Tap: ${userInstructionPrompt}` : userInstructionPrompt
+                    text: userInstructionPrompt
                 }
             ]
         }
@@ -112,68 +109,6 @@ async function AiLocateElement(options) {
         });
         msgs.push(...addOns);
     }
-    if (isAutoGLM(vlMode)) {
-        const { content: rawResponseContent, usage } = await callAIWithStringResponse(msgs, modelConfig);
-        debugInspect('auto-glm rawResponse:', rawResponseContent);
-        const parsed = parseAutoGLMLocateResponse(rawResponseContent);
-        debugInspect('auto-glm thinking:', parsed.think);
-        debugInspect('auto-glm coordinates:', parsed.coordinates);
-        let resRect;
-        let matchedElements = [];
-        let errors = [];
-        if (parsed.error || !parsed.coordinates) {
-            errors = [
-                parsed.error || 'Failed to parse auto-glm response'
-            ];
-            debugInspect('auto-glm parse error:', errors[0]);
-        } else {
-            const { x, y } = parsed.coordinates;
-            debugInspect('auto-glm coordinates [0-999]:', {
-                x,
-                y
-            });
-            const pixelX = Math.round(x * imageWidth / 1000);
-            const pixelY = Math.round(y * imageHeight / 1000);
-            debugInspect('auto-glm pixel coordinates:', {
-                pixelX,
-                pixelY
-            });
-            const bboxSize = 10;
-            const x1 = Math.max(pixelX - bboxSize / 2, 0);
-            const y1 = Math.max(pixelY - bboxSize / 2, 0);
-            const x2 = Math.min(pixelX + bboxSize / 2, imageWidth);
-            const y2 = Math.min(pixelY + bboxSize / 2, imageHeight);
-            resRect = {
-                left: x1,
-                top: y1,
-                width: x2 - x1,
-                height: y2 - y1
-            };
-            if (options.searchConfig?.rect) {
-                resRect.left += options.searchConfig.rect.left;
-                resRect.top += options.searchConfig.rect.top;
-            }
-            debugInspect('auto-glm resRect:', resRect);
-            const rectCenter = {
-                x: resRect.left + resRect.width / 2,
-                y: resRect.top + resRect.height / 2
-            };
-            const element = generateElementByPosition(rectCenter, targetElementDescriptionText);
-            if (element) matchedElements = [
-                element
-            ];
-        }
-        return {
-            rect: resRect,
-            parseResult: {
-                elements: matchedElements,
-                errors
-            },
-            rawResponse: rawResponseContent,
-            usage,
-            reasoning_content: parsed.think
-        };
-    }
     const res = await callAIFn(msgs, modelConfig);
     const rawResponse = JSON.stringify(res.content);
     let resRect;

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ReferenceImage,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { adaptBboxToRect, expandSearchArea, mergeRects } from '../common';\nimport { parseAutoGLMLocateResponse } from './auto-glm/parser';\nimport { getAutoGLMLocatePrompt } from './auto-glm/prompt';\nimport { isAutoGLM } from './auto-glm/util';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  callAIWithObjectResponse,\n  callAIWithStringResponse,\n} from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}':`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const targetElementDescriptionText = extraTextFromUserPrompt(\n    targetElementDescription,\n  );\n  const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n  const systemPrompt = isAutoGLM(vlMode)\n    ? getAutoGLMLocatePrompt(vlMode)\n    : systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: isAutoGLM(vlMode)\n            ? `Tap: ${userInstructionPrompt}`\n            : userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  if (isAutoGLM(vlMode)) {\n    const { content: rawResponseContent, usage } =\n      await callAIWithStringResponse(msgs, modelConfig);\n\n    debugInspect('auto-glm rawResponse:', rawResponseContent);\n\n    const parsed = parseAutoGLMLocateResponse(rawResponseContent);\n\n    debugInspect('auto-glm thinking:', parsed.think);\n    debugInspect('auto-glm coordinates:', parsed.coordinates);\n\n    let resRect: Rect | undefined;\n    let matchedElements: LocateResultElement[] = [];\n    let errors: string[] = [];\n\n    if (parsed.error || !parsed.coordinates) {\n      errors = [parsed.error || 'Failed to parse auto-glm response'];\n      debugInspect('auto-glm parse error:', errors[0]);\n    } else {\n      const { x, y } = parsed.coordinates;\n\n      debugInspect('auto-glm coordinates [0-999]:', { x, y });\n\n      // Convert auto-glm coordinates [0,999] to pixel bbox\n      // Map from [0,999] to pixel coordinates\n      const pixelX = Math.round((x * imageWidth) / 1000);\n      const pixelY = Math.round((y * imageHeight) / 1000);\n\n      debugInspect('auto-glm pixel coordinates:', { pixelX, pixelY });\n\n      // Create a small bbox around the point\n      const bboxSize = 10;\n      const x1 = Math.max(pixelX - bboxSize / 2, 0);\n      const y1 = Math.max(pixelY - bboxSize / 2, 0);\n      const x2 = Math.min(pixelX + bboxSize / 2, imageWidth);\n      const y2 = Math.min(pixelY + bboxSize / 2, imageHeight);\n\n      // Convert to Rect format\n      resRect = {\n        left: x1,\n        top: y1,\n        width: x2 - x1,\n        height: y2 - y1,\n      };\n\n      // Apply offset if searching in a cropped area\n      if (options.searchConfig?.rect) {\n        resRect.left += options.searchConfig.rect.left;\n        resRect.top += options.searchConfig.rect.top;\n      }\n\n      debugInspect('auto-glm resRect:', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n\n    return {\n      rect: resRect,\n      parseResult: {\n        elements: matchedElements,\n        errors,\n      },\n      rawResponse: rawResponseContent,\n      usage,\n      reasoning_content: parsed.think,\n    };\n  }\n\n  const res = await callAIFn(msgs, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: LocateResultElement[] = [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if (\n      'bbox' in res.content &&\n      Array.isArray(res.content.bbox) &&\n      res.content.bbox.length >= 1\n    ) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n      errors = [];\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n    reasoning_content: res.reasoning_content,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen2.5-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n    reasoning_content: result.reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n  modelConfig: IModelConfig,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  const result = await callAIFn(msgs, modelConfig);\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","isAutoGLM","getAutoGLMLocatePrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","paddedResult","paddingToMatchBlockByBase64","addOns","rawResponseContent","usage","callAIWithStringResponse","parsed","parseAutoGLMLocateResponse","resRect","matchedElements","errors","x","y","pixelX","Math","pixelY","bboxSize","x1","y1","x2","y2","rectCenter","element","generateElementByPosition","res","rawResponse","JSON","Array","adaptBboxToRect","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;AAuDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IAUC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,+BAA+BjB,wBACnCW;IAEF,MAAMO,wBAAwBC,kBAAkBF;IAChD,MAAMG,eAAeC,UAAUP,UAC3BQ,uBAAuBR,UACvBS,4BAA4BT;IAEhC,IAAIU,eAAeT;IACnB,IAAIU,aAAaf,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIgB,cAAchB,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIiB,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIjB,QAAQ,YAAY,EAAE;QACxBO,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFe,eAAef,QAAQ,YAAY,CAAC,WAAW;QAC/CgB,aAAahB,QAAQ,YAAY,CAAC,IAAI,EAAE;QACxCiB,cAAcjB,QAAQ,YAAY,CAAC,IAAI,EAAE;QACzCkB,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIZ,AAAW,iBAAXA,QAAyB;QAClC,MAAMe,eAAe,MAAMC,4BAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAMzB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMH,UAAUP,UACZ,CAAC,KAAK,EAAEI,uBAAuB,GAC/BA;gBACN;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOP,0BAAuC;QAChD,MAAMoB,SAAS,MAAM7B,mBAAmB;YACtC,QAAQS,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI2B;IACf;IAEA,IAAIV,UAAUP,SAAS;QACrB,MAAM,EAAE,SAASkB,kBAAkB,EAAEC,KAAK,EAAE,GAC1C,MAAMC,yBAAyB9B,MAAMS;QAEvChB,aAAa,yBAAyBmC;QAEtC,MAAMG,SAASC,2BAA2BJ;QAE1CnC,aAAa,sBAAsBsC,OAAO,KAAK;QAC/CtC,aAAa,yBAAyBsC,OAAO,WAAW;QAExD,IAAIE;QACJ,IAAIC,kBAAyC,EAAE;QAC/C,IAAIC,SAAmB,EAAE;QAEzB,IAAIJ,OAAO,KAAK,IAAI,CAACA,OAAO,WAAW,EAAE;YACvCI,SAAS;gBAACJ,OAAO,KAAK,IAAI;aAAoC;YAC9DtC,aAAa,yBAAyB0C,MAAM,CAAC,EAAE;QACjD,OAAO;YACL,MAAM,EAAEC,CAAC,EAAEC,CAAC,EAAE,GAAGN,OAAO,WAAW;YAEnCtC,aAAa,iCAAiC;gBAAE2C;gBAAGC;YAAE;YAIrD,MAAMC,SAASC,KAAK,KAAK,CAAEH,IAAIf,aAAc;YAC7C,MAAMmB,SAASD,KAAK,KAAK,CAAEF,IAAIf,cAAe;YAE9C7B,aAAa,+BAA+B;gBAAE6C;gBAAQE;YAAO;YAG7D,MAAMC,WAAW;YACjB,MAAMC,KAAKH,KAAK,GAAG,CAACD,SAASG,WAAW,GAAG;YAC3C,MAAME,KAAKJ,KAAK,GAAG,CAACC,SAASC,WAAW,GAAG;YAC3C,MAAMG,KAAKL,KAAK,GAAG,CAACD,SAASG,WAAW,GAAGpB;YAC3C,MAAMwB,KAAKN,KAAK,GAAG,CAACC,SAASC,WAAW,GAAGnB;YAG3CW,UAAU;gBACR,MAAMS;gBACN,KAAKC;gBACL,OAAOC,KAAKF;gBACZ,QAAQG,KAAKF;YACf;YAGA,IAAItC,QAAQ,YAAY,EAAE,MAAM;gBAC9B4B,QAAQ,IAAI,IAAI5B,QAAQ,YAAY,CAAC,IAAI,CAAC,IAAI;gBAC9C4B,QAAQ,GAAG,IAAI5B,QAAQ,YAAY,CAAC,IAAI,CAAC,GAAG;YAC9C;YAEAZ,aAAa,qBAAqBwC;YAElC,MAAMa,aAAa;gBACjB,GAAGb,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMc,UAA+BC,0BACnCF,YACAjC;YAGF,IAAIkC,SACFb,kBAAkB;gBAACa;aAAQ;QAE/B;QAEA,OAAO;YACL,MAAMd;YACN,aAAa;gBACX,UAAUC;gBACVC;YACF;YACA,aAAaP;YACbC;YACA,mBAAmBE,OAAO,KAAK;QACjC;IACF;IAEA,MAAMkB,MAAM,MAAMzC,SAASR,MAAMS;IAEjC,MAAMyC,cAAcC,KAAK,SAAS,CAACF,IAAI,OAAO;IAE9C,IAAIhB;IACJ,IAAIC,kBAAyC,EAAE;IAC/C,IAAIC,SACF,YAAYc,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBG,MAAM,OAAO,CAACH,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;YACAhB,UAAUoB,gBACRJ,IAAI,OAAO,CAAC,IAAI,EAChB5B,YACAC,aACAjB,QAAQ,YAAY,EAAE,MAAM,MAC5BA,QAAQ,YAAY,EAAE,MAAM,KAC5BkB,oBACAC,qBACAd;YAGFjB,aAAa,WAAWwC;YAExB,MAAMa,aAAa;gBACjB,GAAGb,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMc,UAA+BC,0BACnCF,YACAjC;YAEFsB,SAAS,EAAE;YAEX,IAAIY,SACFb,kBAAkB;gBAACa;aAAQ;QAE/B;IACF,EAAE,OAAOO,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACnB,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEoB,IAAI,CAAC,CAAC;aAFtBpB,SAAS;YAACoB;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMtB;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAe;QACA,OAAOD,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAeQ,gBAAgBpD,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEoD,kBAAkB,EAAEjD,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAe2C,4BAA4BjD;IACjD,MAAMkD,gCAAgCC,0BACpCjE,wBAAwB8D;IAE1B,MAAM1D,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMiD;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAM/B,SAAS,MAAM7B,mBAAmB;YACtC,QAAQ4D,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA1D,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMmC,SAAS,MAAMC,yBACnB/D,MACAS;IAGF,IAAIuD;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAab,gBACjBY,aACA3D,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFf,aAAa,0BAA0BuE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DnE,aAAa,wBAAwBwE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASjB,MAAM,OAAO,CAACiB,OAC/B,GAAG,CAAC,CAACA,OACGhB,gBACLgB,MACA/D,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNf,aAAa,qBAAqByE;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DzE,aAAa,iBAAiB2E;QAG9BN,cAAcQ,iBAAiBF,YAAYhE,QAAQ,IAAI,EAAEI;QACzDf,aAAa,2BAA2BqE;IAC1C;IAEA,IAAIS,cAAc9D;IAClB,IAAIqD,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1BhE,kBACAqD,aACAtD,AAAW,iBAAXA;QAEF+D,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAaX,KAAK,SAAS,CAACW,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwBvE,OAO7C;IACC,MAAM,EAAEwE,SAAS,EAAEvE,OAAO,EAAEwE,aAAa,EAAE/E,gBAAgB,EAAEU,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAe+D;IACrB,MAAM,EAAEpE,gBAAgB,EAAE,GAAGL;IAE7B,MAAM0E,wBAAwBC,uBAC5B5E,QAAQ,eAAe,IAAI,IAC3BwE;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKvE;YACL,QAAQ;QACV;IACF;IAGFuE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMhF,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASkE;QACX;KACD;IAED,IAAInF,kBAAkB;QACpB,MAAM4B,SAAS,MAAM7B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMmC,SAAS,MAAMC,yBACnB/D,MACAS;IAEF,OAAO;QACL,aAAaqD,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;QACnB,mBAAmBA,OAAO,iBAAiB;IAC7C;AACF;AAEO,eAAeqB,sBACpBC,WAAmB,EACnB5E,QAAwE,EACxEC,WAAyB;IAKzB,MAAMO,eAAeqE;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAMpF,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASsE;QACX;KACD;IAED,MAAMxB,SAAS,MAAMtD,SAASR,MAAMS;IAEpC,OAAO;QACL,kBAAkBqD,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ReferenceImage,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { adaptBboxToRect, expandSearchArea, mergeRects } from '../common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}':`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const targetElementDescriptionText = extraTextFromUserPrompt(\n    targetElementDescription,\n  );\n  const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: LocateResultElement[] = [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if (\n      'bbox' in res.content &&\n      Array.isArray(res.content.bbox) &&\n      res.content.bbox.length >= 1\n    ) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n      errors = [];\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n    reasoning_content: res.reasoning_content,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen2.5-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n    reasoning_content: result.reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n  modelConfig: IModelConfig,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  const result = await callAIFn(msgs, modelConfig);\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","paddedResult","paddingToMatchBlockByBase64","addOns","res","rawResponse","JSON","resRect","matchedElements","errors","Array","adaptBboxToRect","rectCenter","element","generateElementByPosition","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;AAiDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IAUC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,+BAA+BjB,wBACnCW;IAEF,MAAMO,wBAAwBC,kBAAkBF;IAChD,MAAMG,eAAeC,4BAA4BP;IAEjD,IAAIQ,eAAeP;IACnB,IAAIQ,aAAab,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIc,cAAcd,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIe,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIf,QAAQ,YAAY,EAAE;QACxBO,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFa,eAAeb,QAAQ,YAAY,CAAC,WAAW;QAC/Cc,aAAad,QAAQ,YAAY,CAAC,IAAI,EAAE;QACxCe,cAAcf,QAAQ,YAAY,CAAC,IAAI,EAAE;QACzCgB,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIV,AAAW,iBAAXA,QAAyB;QAClC,MAAMa,eAAe,MAAMC,4BAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAMvB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMJ;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOP,0BAAuC;QAChD,MAAMkB,SAAS,MAAM3B,mBAAmB;YACtC,QAAQS,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMC,MAAM,MAAMlB,SAASR,MAAMS;IAEjC,MAAMkB,cAAcC,KAAK,SAAS,CAACF,IAAI,OAAO;IAE9C,IAAIG;IACJ,IAAIC,kBAAyC,EAAE;IAC/C,IAAIC,SACF,YAAYL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBM,MAAM,OAAO,CAACN,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;YACAG,UAAUI,gBACRP,IAAI,OAAO,CAAC,IAAI,EAChBP,YACAC,aACAf,QAAQ,YAAY,EAAE,MAAM,MAC5BA,QAAQ,YAAY,EAAE,MAAM,KAC5BgB,oBACAC,qBACAZ;YAGFjB,aAAa,WAAWoC;YAExB,MAAMK,aAAa;gBACjB,GAAGL,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMM,UAA+BC,0BACnCF,YACArB;YAEFkB,SAAS,EAAE;YAEX,IAAII,SACFL,kBAAkB;gBAACK;aAAQ;QAE/B;IACF,EAAE,OAAOE,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACN,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;aAFtBP,SAAS;YAACO;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMT;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAJ;QACA,OAAOD,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAec,gBAAgBnC,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEmC,kBAAkB,EAAEhC,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAe0B,4BAA4BhC;IACjD,MAAMiC,gCAAgCC,0BACpChD,wBAAwB6C;IAE1B,MAAMzC,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMgC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMhB,SAAS,MAAM3B,mBAAmB;YACtC,QAAQ2C,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACAzC,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMoB,SAAS,MAAMC,yBACnB9C,MACAS;IAGF,IAAIsC;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAahB,gBACjBe,aACA1C,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFf,aAAa,0BAA0BsD;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DlD,aAAa,wBAAwBuD;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASpB,MAAM,OAAO,CAACoB,OAC/B,GAAG,CAAC,CAACA,OACGnB,gBACLmB,MACA9C,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNf,aAAa,qBAAqBwD;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DxD,aAAa,iBAAiB0D;QAG9BN,cAAcQ,iBAAiBF,YAAY/C,QAAQ,IAAI,EAAEI;QACzDf,aAAa,2BAA2BoD;IAC1C;IAEA,IAAIS,cAAc7C;IAClB,IAAIoC,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1B/C,kBACAoC,aACArC,AAAW,iBAAXA;QAEF8C,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAajB,KAAK,SAAS,CAACiB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwBtD,OAO7C;IACC,MAAM,EAAEuD,SAAS,EAAEtD,OAAO,EAAEuD,aAAa,EAAE9D,gBAAgB,EAAEU,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAe8C;IACrB,MAAM,EAAEnD,gBAAgB,EAAE,GAAGL;IAE7B,MAAMyD,wBAAwBC,uBAC5B3D,QAAQ,eAAe,IAAI,IAC3BuD;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKtD;YACL,QAAQ;QACV;IACF;IAGFsD,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM/D,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASiD;QACX;KACD;IAED,IAAIlE,kBAAkB;QACpB,MAAM0B,SAAS,MAAM3B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMoB,SAAS,MAAMC,yBACnB9C,MACAS;IAEF,OAAO;QACL,aAAaoC,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;QACnB,mBAAmBA,OAAO,iBAAiB;IAC7C;AACF;AAEO,eAAeqB,sBACpBC,WAAmB,EACnB3D,QAAwE,EACxEC,WAAyB;IAKzB,MAAMO,eAAeoD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAMnE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASqD;QACX;KACD;IAED,MAAMxB,SAAS,MAAMrC,SAASR,MAAMS;IAEpC,OAAO;QACL,kBAAkBoC,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/service-caller/index.mjs

@@ -3,7 +3,6 @@ import { getDebug } from "@midscene/shared/logger";
 import { assert, ifInBrowser } from "@midscene/shared/utils";
 import { jsonrepair } from "jsonrepair";
 import openai_0 from "openai";
-import { isAutoGLM } from "../auto-glm/util.mjs";
 async function createChatClient({ modelConfig }) {
     const { socksProxy, httpProxy, modelName, openaiBaseURL, openaiApiKey, openaiExtraConfig, modelDescription, uiTarsModelVersion: uiTarsVersion, vlMode, createOpenAIClient, timeout } = modelConfig;
     let proxyAgent;
@@ -141,10 +140,6 @@ async function callAI(messages, modelConfig, options) {
             vl_high_resolution_images: true
         } : {}
     };
-    if (isAutoGLM(vlMode)) {
-        commonConfig.top_p = 0.85;
-        commonConfig.frequency_penalty = 0.2;
-    }
     const { config: deepThinkConfig, debugMessage, warningMessage } = resolveDeepThinkConfig({
         deepThink: options?.deepThink,
         vlMode
@@ -301,6 +296,14 @@ function resolveDeepThinkConfig({ deepThink, vlMode }) {
         },
         debugMessage: `deepThink mapped to thinking.type=${normalizedDeepThink ? 'enabled' : 'disabled'} for doubao-vision`
     };
+    if ('glm-v' === vlMode) return {
+        config: {
+            thinking: {
+                type: normalizedDeepThink ? 'enabled' : 'disabled'
+            }
+        },
+        debugMessage: `deepThink mapped to thinking.type=${normalizedDeepThink ? 'enabled' : 'disabled'} for glm-v`
+    };
     return {
         config: {},
         debugMessage: `deepThink ignored: unsupported model_family "${vlMode ?? 'default'}"`,

package/dist/es/ai-model/service-caller/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["../../../../src/ai-model/service-caller/index.ts"],"sourcesContent":["import {\n  AIResponseFormat,\n  type AIUsageInfo,\n  type DeepThinkOption,\n} from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_LANGFUSE_DEBUG,\n  MIDSCENE_LANGSMITH_DEBUG,\n  MIDSCENE_MODEL_MAX_TOKENS,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert, ifInBrowser } from '@midscene/shared/utils';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport type { AIArgs } from '../../common';\nimport { isAutoGLM } from '../auto-glm/util';\n\nasync function createChatClient({\n  modelConfig,\n}: {\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n    createOpenAIClient,\n    timeout,\n  } = modelConfig;\n\n  let proxyAgent: any = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n\n  // Helper function to sanitize proxy URL for logging (remove credentials)\n  // Uses URL API instead of regex to avoid ReDoS vulnerabilities\n  const sanitizeProxyUrl = (url: string): string => {\n    try {\n      const parsed = new URL(url);\n      if (parsed.username) {\n        // Keep username for debugging, hide password for security\n        parsed.password = '****';\n        return parsed.href;\n      }\n      return url;\n    } catch {\n      // If URL parsing fails, return original URL (will be caught later)\n      return url;\n    }\n  };\n\n  if (httpProxy) {\n    debugProxy('using http proxy', sanitizeProxyUrl(httpProxy));\n    if (ifInBrowser) {\n      console.warn(\n        'HTTP proxy is configured but not supported in browser environment',\n      );\n    } else {\n      // Dynamic import with variable to avoid bundler static analysis\n      const moduleName = 'undici';\n      const { ProxyAgent } = await import(moduleName);\n      proxyAgent = new ProxyAgent({\n        uri: httpProxy,\n        // Note: authentication is handled via the URI (e.g., http://user:pass@proxy.com:8080)\n      });\n    }\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', sanitizeProxyUrl(socksProxy));\n    if (ifInBrowser) {\n      console.warn(\n        'SOCKS proxy is configured but not supported in browser environment',\n      );\n    } else {\n      try {\n        // Dynamic import with variable to avoid bundler static analysis\n        const moduleName = 'fetch-socks';\n        const { socksDispatcher } = await import(moduleName);\n        // Parse SOCKS proxy URL (e.g., socks5://127.0.0.1:1080)\n        const proxyUrl = new URL(socksProxy);\n\n        // Validate hostname\n        if (!proxyUrl.hostname) {\n          throw new Error('SOCKS proxy URL must include a valid hostname');\n        }\n\n        // Validate and parse port\n        const port = Number.parseInt(proxyUrl.port, 10);\n        if (!proxyUrl.port || Number.isNaN(port)) {\n          throw new Error('SOCKS proxy URL must include a valid port');\n        }\n\n        // Parse SOCKS version from protocol\n        const protocol = proxyUrl.protocol.replace(':', '');\n        const socksType =\n          protocol === 'socks4' ? 4 : protocol === 'socks5' ? 5 : 5;\n\n        proxyAgent = socksDispatcher({\n          type: socksType,\n          host: proxyUrl.hostname,\n          port,\n          ...(proxyUrl.username\n            ? {\n                userId: decodeURIComponent(proxyUrl.username),\n                password: decodeURIComponent(proxyUrl.password || ''),\n              }\n            : {}),\n        });\n        debugProxy('socks proxy configured successfully', {\n          type: socksType,\n          host: proxyUrl.hostname,\n          port: port,\n        });\n      } catch (error) {\n        console.error('Failed to configure SOCKS proxy:', error);\n        throw new Error(\n          `Invalid SOCKS proxy URL: ${socksProxy}. Expected format: socks4://host:port, socks5://host:port, or with authentication: socks5://user:pass@host:port`,\n        );\n      }\n    }\n  }\n\n  const openAIOptions = {\n    baseURL: openaiBaseURL,\n    apiKey: openaiApiKey,\n    // Use fetchOptions.dispatcher for fetch-based SDK instead of httpAgent\n    // Note: Type assertion needed due to undici version mismatch between dependencies\n    ...(proxyAgent ? { fetchOptions: { dispatcher: proxyAgent as any } } : {}),\n    ...openaiExtraConfig,\n    ...(typeof timeout === 'number' ? { timeout } : {}),\n    dangerouslyAllowBrowser: true,\n  };\n\n  const baseOpenAI = new OpenAI(openAIOptions);\n\n  let openai: OpenAI = baseOpenAI;\n\n  // LangSmith wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langsmithModule = 'langsmith/wrappers';\n    const { wrapOpenAI } = await import(langsmithModule);\n    openai = wrapOpenAI(openai);\n  }\n\n  // Langfuse wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGFUSE_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langfuse is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langfuse wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langfuseModule = 'langfuse';\n    const { observeOpenAI } = await import(langfuseModule);\n    openai = observeOpenAI(openai);\n  }\n\n  if (createOpenAIClient) {\n    const wrappedClient = await createOpenAIClient(baseOpenAI, openAIOptions);\n\n    if (wrappedClient) {\n      openai = wrappedClient as OpenAI;\n    }\n  }\n\n  return {\n    completion: openai.chat.completions,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  };\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n    deepThink?: DeepThinkOption;\n  },\n): Promise<{\n  content: string;\n  reasoning_content?: string;\n  usage?: AIUsageInfo;\n  isStreamed: boolean;\n}> {\n  const { completion, modelName, modelDescription, uiTarsVersion, vlMode } =\n    await createChatClient({\n      modelConfig,\n    });\n\n  const maxTokens =\n    globalConfigManager.getEnvConfigValueAsNumber(MIDSCENE_MODEL_MAX_TOKENS) ??\n    globalConfigManager.getEnvConfigValueAsNumber(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n  const temperature = modelConfig.temperature ?? 0;\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let accumulatedReasoning = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const buildUsageInfo = (usageData?: OpenAI.CompletionUsage) => {\n    if (!usageData) return undefined;\n\n    const cachedInputTokens = (\n      usageData as { prompt_tokens_details?: { cached_tokens?: number } }\n    )?.prompt_tokens_details?.cached_tokens;\n\n    return {\n      prompt_tokens: usageData.prompt_tokens ?? 0,\n      completion_tokens: usageData.completion_tokens ?? 0,\n      total_tokens: usageData.total_tokens ?? 0,\n      cached_input: cachedInputTokens ?? 0,\n      time_cost: timeCost ?? 0,\n      model_name: modelName,\n      model_description: modelDescription,\n      intent: modelConfig.intent,\n    } satisfies AIUsageInfo;\n  };\n\n  const commonConfig = {\n    temperature,\n    stream: !!isStreaming,\n    max_tokens: maxTokens,\n    ...(vlMode === 'qwen2.5-vl' // qwen vl v2 specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  if (isAutoGLM(vlMode)) {\n    (commonConfig as unknown as Record<string, number>).top_p = 0.85;\n    (commonConfig as unknown as Record<string, number>).frequency_penalty = 0.2;\n  }\n\n  const {\n    config: deepThinkConfig,\n    debugMessage,\n    warningMessage,\n  } = resolveDeepThinkConfig({\n    deepThink: options?.deepThink,\n    vlMode,\n  });\n  if (debugMessage) {\n    debugCall(debugMessage);\n  }\n  if (warningMessage) {\n    debugCall(warningMessage);\n    console.warn(warningMessage);\n  }\n\n  try {\n    debugCall(\n      `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n    );\n\n    if (isStreaming) {\n      const stream = (await completion.create(\n        {\n          model: modelName,\n          messages,\n          ...commonConfig,\n          ...deepThinkConfig,\n        },\n        {\n          stream: true,\n        },\n      )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n        _request_id?: string | null;\n      };\n\n      for await (const chunk of stream) {\n        const content = chunk.choices?.[0]?.delta?.content || '';\n        const reasoning_content =\n          (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n        // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n        if (chunk.usage) {\n          usage = chunk.usage;\n        }\n\n        if (content || reasoning_content) {\n          accumulated += content;\n          accumulatedReasoning += reasoning_content;\n          const chunkData: CodeGenerationChunk = {\n            content,\n            reasoning_content,\n            accumulated,\n            isComplete: false,\n            usage: undefined,\n          };\n          options.onChunk!(chunkData);\n        }\n\n        // Check if stream is complete\n        if (chunk.choices?.[0]?.finish_reason) {\n          timeCost = Date.now() - startTime;\n\n          // If usage is not available from the stream, provide a basic usage info\n          if (!usage) {\n            // Estimate token counts based on content length (rough approximation)\n            const estimatedTokens = Math.max(\n              1,\n              Math.floor(accumulated.length / 4),\n            );\n            usage = {\n              prompt_tokens: estimatedTokens,\n              completion_tokens: estimatedTokens,\n              total_tokens: estimatedTokens * 2,\n            };\n          }\n\n          // Send final chunk\n          const finalChunk: CodeGenerationChunk = {\n            content: '',\n            accumulated,\n            reasoning_content: '',\n            isComplete: true,\n            usage: buildUsageInfo(usage),\n          };\n          options.onChunk!(finalChunk);\n          break;\n        }\n      }\n      content = accumulated;\n      debugProfileStats(\n        `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}, temperature, ${temperature ?? ''}`,\n      );\n    } else {\n      const result = await completion.create({\n        model: modelName,\n        messages,\n        ...commonConfig,\n        ...deepThinkConfig,\n      } as any);\n      timeCost = Date.now() - startTime;\n\n      debugProfileStats(\n        `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}, temperature, ${temperature ?? ''}`,\n      );\n\n      debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);\n\n      assert(\n        result.choices,\n        `invalid response from LLM service: ${JSON.stringify(result)}`,\n      );\n      content = result.choices[0].message.content!;\n      accumulatedReasoning =\n        (result.choices[0].message as any)?.reasoning_content || '';\n      usage = result.usage;\n    }\n\n    debugCall(`response reasoning content: ${accumulatedReasoning}`);\n    debugCall(`response content: ${content}`);\n    assert(content, 'empty content');\n\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      } as OpenAI.CompletionUsage;\n    }\n\n    return {\n      content: content || '',\n      reasoning_content: accumulatedReasoning || undefined,\n      usage: buildUsageInfo(usage),\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service (${modelName}): ${e.message}\\nTrouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport async function callAIWithObjectResponse<T>(\n  messages: ChatCompletionMessageParam[],\n  modelConfig: IModelConfig,\n  options?: {\n    deepThink?: DeepThinkOption;\n  },\n): Promise<{\n  content: T;\n  contentString: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const response = await callAI(messages, modelConfig, {\n    deepThink: options?.deepThink,\n  });\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  assert(\n    typeof jsonContent === 'object',\n    `failed to parse json response from model (${modelConfig.modelName}): ${response.content}`,\n  );\n  return {\n    content: jsonContent,\n    contentString: response.content,\n    usage: response.usage,\n    reasoning_content: response.reasoning_content,\n  };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function resolveDeepThinkConfig({\n  deepThink,\n  vlMode,\n}: {\n  deepThink?: DeepThinkOption;\n  vlMode?: TVlModeTypes;\n}): {\n  config: Record<string, unknown>;\n  debugMessage?: string;\n  warningMessage?: string;\n} {\n  const normalizedDeepThink = deepThink === 'unset' ? undefined : deepThink;\n\n  if (normalizedDeepThink === undefined) {\n    return { config: {}, debugMessage: undefined };\n  }\n\n  if (vlMode === 'qwen3-vl') {\n    return {\n      config: { enable_thinking: normalizedDeepThink },\n      debugMessage: `deepThink mapped to enable_thinking=${normalizedDeepThink} for qwen3-vl`,\n    };\n  }\n\n  if (vlMode === 'doubao-vision') {\n    return {\n      config: {\n        thinking: { type: normalizedDeepThink ? 'enabled' : 'disabled' },\n      },\n      debugMessage: `deepThink mapped to thinking.type=${normalizedDeepThink ? 'enabled' : 'disabled'} for doubao-vision`,\n    };\n  }\n\n  return {\n    config: {},\n    debugMessage: `deepThink ignored: unsupported model_family \"${vlMode ?? 'default'}\"`,\n    warningMessage: `The \"deepThink\" option is not supported for model_family \"${vlMode ?? 'default'}\".`,\n  };\n}\n\n/**\n * Normalize a parsed JSON object by trimming whitespace from:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction normalizeJsonObject(obj: any): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeJsonObject(item));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n\n      // Recursively normalize the value\n      let normalizedValue = normalizeJsonObject(value);\n\n      // Trim all string values\n      if (typeof normalizedValue === 'string') {\n        normalizedValue = normalizedValue.trim();\n      }\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n\n  let parsed: any;\n  let lastError: unknown;\n  try {\n    parsed = JSON.parse(cleanJsonString);\n    return normalizeJsonObject(parsed);\n  } catch (error) {\n    lastError = error;\n  }\n  try {\n    parsed = JSON.parse(jsonrepair(cleanJsonString));\n    return normalizeJsonObject(parsed);\n  } catch (error) {\n    lastError = error;\n  }\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    try {\n      parsed = JSON.parse(jsonrepair(jsonString));\n      return normalizeJsonObject(parsed);\n    } catch (error) {\n      lastError = error;\n    }\n  }\n  throw Error(\n    `failed to parse LLM response into JSON. Error - ${String(\n      lastError ?? 'unknown error',\n    )}. Response - \\n ${input}`,\n  );\n}\n"],"names":["createChatClient","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","modelDescription","uiTarsVersion","vlMode","createOpenAIClient","timeout","proxyAgent","debugProxy","getDebug","sanitizeProxyUrl","url","parsed","URL","ifInBrowser","console","moduleName","ProxyAgent","socksDispatcher","proxyUrl","Error","port","Number","protocol","socksType","decodeURIComponent","error","openAIOptions","baseOpenAI","OpenAI","openai","globalConfigManager","MIDSCENE_LANGSMITH_DEBUG","langsmithModule","wrapOpenAI","MIDSCENE_LANGFUSE_DEBUG","langfuseModule","observeOpenAI","wrappedClient","callAI","messages","options","completion","maxTokens","MIDSCENE_MODEL_MAX_TOKENS","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","temperature","isStreaming","content","accumulated","accumulatedReasoning","usage","timeCost","buildUsageInfo","usageData","cachedInputTokens","commonConfig","isAutoGLM","deepThinkConfig","debugMessage","warningMessage","resolveDeepThinkConfig","stream","chunk","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","result","JSON","assert","e","newError","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","deepThink","normalizedDeepThink","normalizeJsonObject","obj","Array","item","normalized","key","value","Object","trimmedKey","normalizedValue","cleanJsonString","lastError","jsonrepair","jsonString","String"],"mappings":";;;;;;AA0BA,eAAeA,iBAAiB,EAC9BC,WAAW,EAGZ;IAOC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACNC,kBAAkB,EAClBC,OAAO,EACR,GAAGX;IAEJ,IAAIY;IACJ,MAAMC,aAAaC,SAAS;IAI5B,MAAMC,mBAAmB,CAACC;QACxB,IAAI;YACF,MAAMC,SAAS,IAAIC,IAAIF;YACvB,IAAIC,OAAO,QAAQ,EAAE;gBAEnBA,OAAO,QAAQ,GAAG;gBAClB,OAAOA,OAAO,IAAI;YACpB;YACA,OAAOD;QACT,EAAE,OAAM;YAEN,OAAOA;QACT;IACF;IAEA,IAAId,WAAW;QACbW,WAAW,oBAAoBE,iBAAiBb;QAChD,IAAIiB,aACFC,QAAQ,IAAI,CACV;aAEG;YAEL,MAAMC,aAAa;YACnB,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAACD;YACpCT,aAAa,IAAIU,WAAW;gBAC1B,KAAKpB;YAEP;QACF;IACF,OAAO,IAAID,YAAY;QACrBY,WAAW,qBAAqBE,iBAAiBd;QACjD,IAAIkB,aACFC,QAAQ,IAAI,CACV;aAGF,IAAI;YAEF,MAAMC,aAAa;YACnB,MAAM,EAAEE,eAAe,EAAE,GAAG,MAAM,MAAM,CAACF;YAEzC,MAAMG,WAAW,IAAIN,IAAIjB;YAGzB,IAAI,CAACuB,SAAS,QAAQ,EACpB,MAAM,IAAIC,MAAM;YAIlB,MAAMC,OAAOC,OAAO,QAAQ,CAACH,SAAS,IAAI,EAAE;YAC5C,IAAI,CAACA,SAAS,IAAI,IAAIG,OAAO,KAAK,CAACD,OACjC,MAAM,IAAID,MAAM;YAIlB,MAAMG,WAAWJ,SAAS,QAAQ,CAAC,OAAO,CAAC,KAAK;YAChD,MAAMK,YACJD,AAAa,aAAbA,WAAwB,IAAIA,AAAa,aAAbA,WAAwB,IAAI;YAE1DhB,aAAaW,gBAAgB;gBAC3B,MAAMM;gBACN,MAAML,SAAS,QAAQ;gBACvBE;gBACA,GAAIF,SAAS,QAAQ,GACjB;oBACE,QAAQM,mBAAmBN,SAAS,QAAQ;oBAC5C,UAAUM,mBAAmBN,SAAS,QAAQ,IAAI;gBACpD,IACA,CAAC,CAAC;YACR;YACAX,WAAW,uCAAuC;gBAChD,MAAMgB;gBACN,MAAML,SAAS,QAAQ;gBACvB,MAAME;YACR;QACF,EAAE,OAAOK,OAAO;YACdX,QAAQ,KAAK,CAAC,oCAAoCW;YAClD,MAAM,IAAIN,MACR,CAAC,yBAAyB,EAAExB,WAAW,+GAA+G,CAAC;QAE3J;IAEJ;IAEA,MAAM+B,gBAAgB;QACpB,SAAS5B;QACT,QAAQC;QAGR,GAAIO,aAAa;YAAE,cAAc;gBAAE,YAAYA;YAAkB;QAAE,IAAI,CAAC,CAAC;QACzE,GAAGN,iBAAiB;QACpB,GAAI,AAAmB,YAAnB,OAAOK,UAAuB;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAClD,yBAAyB;IAC3B;IAEA,MAAMsB,aAAa,IAAIC,SAAOF;IAE9B,IAAIG,SAAiBF;IAGrB,IACEE,UACAC,oBAAoB,qBAAqB,CAACC,2BAC1C;QACA,IAAIlB,aACF,MAAM,IAAIM,MAAM;QAElBL,QAAQ,GAAG,CAAC;QAEZ,MAAMkB,kBAAkB;QACxB,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAACD;QACpCH,SAASI,WAAWJ;IACtB;IAGA,IACEA,UACAC,oBAAoB,qBAAqB,CAACI,0BAC1C;QACA,IAAIrB,aACF,MAAM,IAAIM,MAAM;QAElBL,QAAQ,GAAG,CAAC;QAEZ,MAAMqB,iBAAiB;QACvB,MAAM,EAAEC,aAAa,EAAE,GAAG,MAAM,MAAM,CAACD;QACvCN,SAASO,cAAcP;IACzB;IAEA,IAAIzB,oBAAoB;QACtB,MAAMiC,gBAAgB,MAAMjC,mBAAmBuB,YAAYD;QAE3D,IAAIW,eACFR,SAASQ;IAEb;IAEA,OAAO;QACL,YAAYR,OAAO,IAAI,CAAC,WAAW;QACnChC;QACAI;QACAC;QACAC;IACF;AACF;AAEO,eAAemC,OACpBC,QAAsC,EACtC7C,WAAyB,EACzB8C,OAIC;IAOD,MAAM,EAAEC,UAAU,EAAE5C,SAAS,EAAEI,gBAAgB,EAAEC,aAAa,EAAEC,MAAM,EAAE,GACtE,MAAMV,iBAAiB;QACrBC;IACF;IAEF,MAAMgD,YACJZ,oBAAoB,yBAAyB,CAACa,8BAC9Cb,oBAAoB,yBAAyB,CAACc;IAChD,MAAMC,YAAYrC,SAAS;IAC3B,MAAMsC,oBAAoBtC,SAAS;IACnC,MAAMuC,qBAAqBvC,SAAS;IAEpC,MAAMwC,YAAYC,KAAK,GAAG;IAC1B,MAAMC,cAAcxD,YAAY,WAAW,IAAI;IAE/C,MAAMyD,cAAcX,SAAS,UAAUA,SAAS;IAChD,IAAIY;IACJ,IAAIC,cAAc;IAClB,IAAIC,uBAAuB;IAC3B,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,iBAAiB,CAACC;QACtB,IAAI,CAACA,WAAW;QAEhB,MAAMC,oBACJD,WACC,uBAAuB;QAE1B,OAAO;YACL,eAAeA,UAAU,aAAa,IAAI;YAC1C,mBAAmBA,UAAU,iBAAiB,IAAI;YAClD,cAAcA,UAAU,YAAY,IAAI;YACxC,cAAcC,qBAAqB;YACnC,WAAWH,YAAY;YACvB,YAAY3D;YACZ,mBAAmBI;YACnB,QAAQP,YAAY,MAAM;QAC5B;IACF;IAEA,MAAMkE,eAAe;QACnBV;QACA,QAAQ,CAAC,CAACC;QACV,YAAYT;QACZ,GAAIvC,AAAW,iBAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI0D,UAAU1D,SAAS;QACpByD,aAAmD,KAAK,GAAG;QAC3DA,aAAmD,iBAAiB,GAAG;IAC1E;IAEA,MAAM,EACJ,QAAQE,eAAe,EACvBC,YAAY,EACZC,cAAc,EACf,GAAGC,uBAAuB;QACzB,WAAWzB,SAAS;QACpBrC;IACF;IACA,IAAI4D,cACFlB,UAAUkB;IAEZ,IAAIC,gBAAgB;QAClBnB,UAAUmB;QACVlD,QAAQ,IAAI,CAACkD;IACf;IAEA,IAAI;QACFnB,UACE,CAAC,QAAQ,EAAEM,cAAc,eAAe,GAAG,WAAW,EAAEtD,WAAW;QAGrE,IAAIsD,aAAa;YACf,MAAMe,SAAU,MAAMzB,WAAW,MAAM,CACrC;gBACE,OAAO5C;gBACP0C;gBACA,GAAGqB,YAAY;gBACf,GAAGE,eAAe;YACpB,GACA;gBACE,QAAQ;YACV;YAKF,WAAW,MAAMK,SAASD,OAAQ;gBAChC,MAAMd,UAAUe,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,OAAO,WAAW;gBACtD,MAAMC,oBACHD,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,OAAe,qBAAqB;gBAG3D,IAAIA,MAAM,KAAK,EACbZ,QAAQY,MAAM,KAAK;gBAGrB,IAAIf,WAAWgB,mBAAmB;oBAChCf,eAAeD;oBACfE,wBAAwBc;oBACxB,MAAMC,YAAiC;wBACrCjB;wBACAgB;wBACAf;wBACA,YAAY;wBACZ,OAAOiB;oBACT;oBACA9B,QAAQ,OAAO,CAAE6B;gBACnB;gBAGA,IAAIF,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,eAAe;oBACrCX,WAAWP,KAAK,GAAG,KAAKD;oBAGxB,IAAI,CAACO,OAAO;wBAEV,MAAMgB,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACnB,YAAY,MAAM,GAAG;wBAElCE,QAAQ;4BACN,eAAegB;4BACf,mBAAmBA;4BACnB,cAAcA,AAAkB,IAAlBA;wBAChB;oBACF;oBAGA,MAAME,aAAkC;wBACtC,SAAS;wBACTpB;wBACA,mBAAmB;wBACnB,YAAY;wBACZ,OAAOI,eAAeF;oBACxB;oBACAf,QAAQ,OAAO,CAAEiC;oBACjB;gBACF;YACF;YACArB,UAAUC;YACVP,kBACE,CAAC,iBAAiB,EAAEjD,UAAU,QAAQ,EAAEM,UAAU,UAAU,WAAW,EAAEqD,SAAS,eAAe,EAAEN,eAAe,IAAI;QAE1H,OAAO;YACL,MAAMwB,SAAS,MAAMjC,WAAW,MAAM,CAAC;gBACrC,OAAO5C;gBACP0C;gBACA,GAAGqB,YAAY;gBACf,GAAGE,eAAe;YACpB;YACAN,WAAWP,KAAK,GAAG,KAAKD;YAExBF,kBACE,CAAC,OAAO,EAAEjD,UAAU,QAAQ,EAAEM,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAEwE,OAAO,KAAK,EAAE,iBAAiB,GAAG,qBAAqB,EAAEA,OAAO,KAAK,EAAE,qBAAqB,GAAG,gBAAgB,EAAEA,OAAO,KAAK,EAAE,gBAAgB,GAAG,WAAW,EAAElB,SAAS,aAAa,EAAEkB,OAAO,WAAW,IAAI,GAAG,eAAe,EAAExB,eAAe,IAAI;YAG9VH,mBAAmB,CAAC,oBAAoB,EAAE4B,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;YAExEE,OACEF,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;YAEhEtB,UAAUsB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;YAC3CpB,uBACGoB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,EAAU,qBAAqB;YAC3DnB,QAAQmB,OAAO,KAAK;QACtB;QAEA7B,UAAU,CAAC,4BAA4B,EAAES,sBAAsB;QAC/DT,UAAU,CAAC,kBAAkB,EAAEO,SAAS;QACxCwB,OAAOxB,SAAS;QAGhB,IAAID,eAAe,CAACI,OAAO;YAEzB,MAAMgB,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEpB,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCG,QAAQ;gBACN,eAAegB;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASnB,WAAW;YACpB,mBAAmBE,wBAAwBgB;YAC3C,OAAOb,eAAeF;YACtB,YAAY,CAAC,CAACJ;QAChB;IACF,EAAE,OAAO0B,GAAQ;QACf/D,QAAQ,KAAK,CAAC,kBAAkB+D;QAChC,MAAMC,WAAW,IAAI3D,MACnB,CAAC,eAAe,EAAEgC,cAAc,eAAe,GAAG,kBAAkB,EAAEtD,UAAU,GAAG,EAAEgF,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC9J;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,eAAeC,yBACpBxC,QAAsC,EACtC7C,WAAyB,EACzB8C,OAEC;IAOD,MAAMwC,WAAW,MAAM1C,OAAOC,UAAU7C,aAAa;QACnD,WAAW8C,SAAS;IACtB;IACAoC,OAAOI,UAAU;IACjB,MAAM7E,SAAST,YAAY,MAAM;IACjC,MAAMuF,cAAcC,cAAcF,SAAS,OAAO,EAAE7E;IACpDyE,OACE,AAAuB,YAAvB,OAAOK,aACP,CAAC,0CAA0C,EAAEvF,YAAY,SAAS,CAAC,GAAG,EAAEsF,SAAS,OAAO,EAAE;IAE5F,OAAO;QACL,SAASC;QACT,eAAeD,SAAS,OAAO;QAC/B,OAAOA,SAAS,KAAK;QACrB,mBAAmBA,SAAS,iBAAiB;IAC/C;AACF;AAEO,eAAeG,yBACpBC,IAAY,EACZ1F,WAAyB;IAEzB,MAAM,EAAE0D,OAAO,EAAEG,KAAK,EAAE,GAAG,MAAMjB,OAAO8C,MAAM1F;IAC9C,OAAO;QAAE0D;QAASG;IAAM;AAC1B;AAEO,SAAS8B,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASzB,uBAAuB,EACrC0B,SAAS,EACTxF,MAAM,EAIP;IAKC,MAAMyF,sBAAsBD,AAAc,YAAdA,YAAwBrB,SAAYqB;IAEhE,IAAIC,AAAwBtB,WAAxBsB,qBACF,OAAO;QAAE,QAAQ,CAAC;QAAG,cAActB;IAAU;IAG/C,IAAInE,AAAW,eAAXA,QACF,OAAO;QACL,QAAQ;YAAE,iBAAiByF;QAAoB;QAC/C,cAAc,CAAC,oCAAoC,EAAEA,oBAAoB,aAAa,CAAC;IACzF;IAGF,IAAIzF,AAAW,oBAAXA,QACF,OAAO;QACL,QAAQ;YACN,UAAU;gBAAE,MAAMyF,sBAAsB,YAAY;YAAW;QACjE;QACA,cAAc,CAAC,kCAAkC,EAAEA,sBAAsB,YAAY,WAAW,kBAAkB,CAAC;IACrH;IAGF,OAAO;QACL,QAAQ,CAAC;QACT,cAAc,CAAC,6CAA6C,EAAEzF,UAAU,UAAU,CAAC,CAAC;QACpF,gBAAgB,CAAC,0DAA0D,EAAEA,UAAU,UAAU,EAAE,CAAC;IACtG;AACF;AAQA,SAAS0F,oBAAoBC,GAAQ;IAEnC,IAAIA,QAAAA,KACF,OAAOA;IAIT,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACE,OAASH,oBAAoBG;IAI/C,IAAI,AAAe,YAAf,OAAOF,KAAkB;QAC3B,MAAMG,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACN,KAAM;YAE9C,MAAMO,aAAaH,IAAI,IAAI;YAG3B,IAAII,kBAAkBT,oBAAoBM;YAG1C,IAAI,AAA2B,YAA3B,OAAOG,iBACTA,kBAAkBA,gBAAgB,IAAI;YAGxCL,UAAU,CAACI,WAAW,GAAGC;QAC3B;QAEA,OAAOL;IACT;IAGA,IAAI,AAAe,YAAf,OAAOH,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEO,SAASZ,cAAcQ,KAAa,EAAEvF,MAAgC;IAC3E,MAAMoG,kBAAkBlB,yBAAyBK;IAEjD,IAAIa,iBAAiB,MAAM,oBACzB,OAAOA,gBACJ,KAAK,CAAC,oBACL,MAAM,GACP,IAAIlF;IAGT,IAAIV;IACJ,IAAI6F;IACJ,IAAI;QACF7F,SAASgE,KAAK,KAAK,CAAC4B;QACpB,OAAOV,oBAAoBlF;IAC7B,EAAE,OAAOc,OAAO;QACd+E,YAAY/E;IACd;IACA,IAAI;QACFd,SAASgE,KAAK,KAAK,CAAC8B,WAAWF;QAC/B,OAAOV,oBAAoBlF;IAC7B,EAAE,OAAOc,OAAO;QACd+E,YAAY/E;IACd;IAEA,IAAItB,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAMuG,aAAajB,yBAAyBc;QAC5C,IAAI;YACF5F,SAASgE,KAAK,KAAK,CAAC8B,WAAWC;YAC/B,OAAOb,oBAAoBlF;QAC7B,EAAE,OAAOc,OAAO;YACd+E,YAAY/E;QACd;IACF;IACA,MAAMN,MACJ,CAAC,gDAAgD,EAAEwF,OACjDH,aAAa,iBACb,gBAAgB,EAAEd,OAAO;AAE/B"}
+{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["../../../../src/ai-model/service-caller/index.ts"],"sourcesContent":["import {\n  AIResponseFormat,\n  type AIUsageInfo,\n  type DeepThinkOption,\n} from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_LANGFUSE_DEBUG,\n  MIDSCENE_LANGSMITH_DEBUG,\n  MIDSCENE_MODEL_MAX_TOKENS,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert, ifInBrowser } from '@midscene/shared/utils';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport type { AIArgs } from '../../common';\n\nasync function createChatClient({\n  modelConfig,\n}: {\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n    createOpenAIClient,\n    timeout,\n  } = modelConfig;\n\n  let proxyAgent: any = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n\n  // Helper function to sanitize proxy URL for logging (remove credentials)\n  // Uses URL API instead of regex to avoid ReDoS vulnerabilities\n  const sanitizeProxyUrl = (url: string): string => {\n    try {\n      const parsed = new URL(url);\n      if (parsed.username) {\n        // Keep username for debugging, hide password for security\n        parsed.password = '****';\n        return parsed.href;\n      }\n      return url;\n    } catch {\n      // If URL parsing fails, return original URL (will be caught later)\n      return url;\n    }\n  };\n\n  if (httpProxy) {\n    debugProxy('using http proxy', sanitizeProxyUrl(httpProxy));\n    if (ifInBrowser) {\n      console.warn(\n        'HTTP proxy is configured but not supported in browser environment',\n      );\n    } else {\n      // Dynamic import with variable to avoid bundler static analysis\n      const moduleName = 'undici';\n      const { ProxyAgent } = await import(moduleName);\n      proxyAgent = new ProxyAgent({\n        uri: httpProxy,\n        // Note: authentication is handled via the URI (e.g., http://user:pass@proxy.com:8080)\n      });\n    }\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', sanitizeProxyUrl(socksProxy));\n    if (ifInBrowser) {\n      console.warn(\n        'SOCKS proxy is configured but not supported in browser environment',\n      );\n    } else {\n      try {\n        // Dynamic import with variable to avoid bundler static analysis\n        const moduleName = 'fetch-socks';\n        const { socksDispatcher } = await import(moduleName);\n        // Parse SOCKS proxy URL (e.g., socks5://127.0.0.1:1080)\n        const proxyUrl = new URL(socksProxy);\n\n        // Validate hostname\n        if (!proxyUrl.hostname) {\n          throw new Error('SOCKS proxy URL must include a valid hostname');\n        }\n\n        // Validate and parse port\n        const port = Number.parseInt(proxyUrl.port, 10);\n        if (!proxyUrl.port || Number.isNaN(port)) {\n          throw new Error('SOCKS proxy URL must include a valid port');\n        }\n\n        // Parse SOCKS version from protocol\n        const protocol = proxyUrl.protocol.replace(':', '');\n        const socksType =\n          protocol === 'socks4' ? 4 : protocol === 'socks5' ? 5 : 5;\n\n        proxyAgent = socksDispatcher({\n          type: socksType,\n          host: proxyUrl.hostname,\n          port,\n          ...(proxyUrl.username\n            ? {\n                userId: decodeURIComponent(proxyUrl.username),\n                password: decodeURIComponent(proxyUrl.password || ''),\n              }\n            : {}),\n        });\n        debugProxy('socks proxy configured successfully', {\n          type: socksType,\n          host: proxyUrl.hostname,\n          port: port,\n        });\n      } catch (error) {\n        console.error('Failed to configure SOCKS proxy:', error);\n        throw new Error(\n          `Invalid SOCKS proxy URL: ${socksProxy}. Expected format: socks4://host:port, socks5://host:port, or with authentication: socks5://user:pass@host:port`,\n        );\n      }\n    }\n  }\n\n  const openAIOptions = {\n    baseURL: openaiBaseURL,\n    apiKey: openaiApiKey,\n    // Use fetchOptions.dispatcher for fetch-based SDK instead of httpAgent\n    // Note: Type assertion needed due to undici version mismatch between dependencies\n    ...(proxyAgent ? { fetchOptions: { dispatcher: proxyAgent as any } } : {}),\n    ...openaiExtraConfig,\n    ...(typeof timeout === 'number' ? { timeout } : {}),\n    dangerouslyAllowBrowser: true,\n  };\n\n  const baseOpenAI = new OpenAI(openAIOptions);\n\n  let openai: OpenAI = baseOpenAI;\n\n  // LangSmith wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langsmithModule = 'langsmith/wrappers';\n    const { wrapOpenAI } = await import(langsmithModule);\n    openai = wrapOpenAI(openai);\n  }\n\n  // Langfuse wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGFUSE_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langfuse is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langfuse wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langfuseModule = 'langfuse';\n    const { observeOpenAI } = await import(langfuseModule);\n    openai = observeOpenAI(openai);\n  }\n\n  if (createOpenAIClient) {\n    const wrappedClient = await createOpenAIClient(baseOpenAI, openAIOptions);\n\n    if (wrappedClient) {\n      openai = wrappedClient as OpenAI;\n    }\n  }\n\n  return {\n    completion: openai.chat.completions,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  };\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n    deepThink?: DeepThinkOption;\n  },\n): Promise<{\n  content: string;\n  reasoning_content?: string;\n  usage?: AIUsageInfo;\n  isStreamed: boolean;\n}> {\n  const { completion, modelName, modelDescription, uiTarsVersion, vlMode } =\n    await createChatClient({\n      modelConfig,\n    });\n\n  const maxTokens =\n    globalConfigManager.getEnvConfigValueAsNumber(MIDSCENE_MODEL_MAX_TOKENS) ??\n    globalConfigManager.getEnvConfigValueAsNumber(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n  const temperature = modelConfig.temperature ?? 0;\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let accumulatedReasoning = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const buildUsageInfo = (usageData?: OpenAI.CompletionUsage) => {\n    if (!usageData) return undefined;\n\n    const cachedInputTokens = (\n      usageData as { prompt_tokens_details?: { cached_tokens?: number } }\n    )?.prompt_tokens_details?.cached_tokens;\n\n    return {\n      prompt_tokens: usageData.prompt_tokens ?? 0,\n      completion_tokens: usageData.completion_tokens ?? 0,\n      total_tokens: usageData.total_tokens ?? 0,\n      cached_input: cachedInputTokens ?? 0,\n      time_cost: timeCost ?? 0,\n      model_name: modelName,\n      model_description: modelDescription,\n      intent: modelConfig.intent,\n    } satisfies AIUsageInfo;\n  };\n\n  const commonConfig = {\n    temperature,\n    stream: !!isStreaming,\n    max_tokens: maxTokens,\n    ...(vlMode === 'qwen2.5-vl' // qwen vl v2 specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n  const {\n    config: deepThinkConfig,\n    debugMessage,\n    warningMessage,\n  } = resolveDeepThinkConfig({\n    deepThink: options?.deepThink,\n    vlMode,\n  });\n  if (debugMessage) {\n    debugCall(debugMessage);\n  }\n  if (warningMessage) {\n    debugCall(warningMessage);\n    console.warn(warningMessage);\n  }\n\n  try {\n    debugCall(\n      `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n    );\n\n    if (isStreaming) {\n      const stream = (await completion.create(\n        {\n          model: modelName,\n          messages,\n          ...commonConfig,\n          ...deepThinkConfig,\n        },\n        {\n          stream: true,\n        },\n      )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n        _request_id?: string | null;\n      };\n\n      for await (const chunk of stream) {\n        const content = chunk.choices?.[0]?.delta?.content || '';\n        const reasoning_content =\n          (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n        // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n        if (chunk.usage) {\n          usage = chunk.usage;\n        }\n\n        if (content || reasoning_content) {\n          accumulated += content;\n          accumulatedReasoning += reasoning_content;\n          const chunkData: CodeGenerationChunk = {\n            content,\n            reasoning_content,\n            accumulated,\n            isComplete: false,\n            usage: undefined,\n          };\n          options.onChunk!(chunkData);\n        }\n\n        // Check if stream is complete\n        if (chunk.choices?.[0]?.finish_reason) {\n          timeCost = Date.now() - startTime;\n\n          // If usage is not available from the stream, provide a basic usage info\n          if (!usage) {\n            // Estimate token counts based on content length (rough approximation)\n            const estimatedTokens = Math.max(\n              1,\n              Math.floor(accumulated.length / 4),\n            );\n            usage = {\n              prompt_tokens: estimatedTokens,\n              completion_tokens: estimatedTokens,\n              total_tokens: estimatedTokens * 2,\n            };\n          }\n\n          // Send final chunk\n          const finalChunk: CodeGenerationChunk = {\n            content: '',\n            accumulated,\n            reasoning_content: '',\n            isComplete: true,\n            usage: buildUsageInfo(usage),\n          };\n          options.onChunk!(finalChunk);\n          break;\n        }\n      }\n      content = accumulated;\n      debugProfileStats(\n        `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}, temperature, ${temperature ?? ''}`,\n      );\n    } else {\n      const result = await completion.create({\n        model: modelName,\n        messages,\n        ...commonConfig,\n        ...deepThinkConfig,\n      } as any);\n      timeCost = Date.now() - startTime;\n\n      debugProfileStats(\n        `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}, temperature, ${temperature ?? ''}`,\n      );\n\n      debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);\n\n      assert(\n        result.choices,\n        `invalid response from LLM service: ${JSON.stringify(result)}`,\n      );\n      content = result.choices[0].message.content!;\n      accumulatedReasoning =\n        (result.choices[0].message as any)?.reasoning_content || '';\n      usage = result.usage;\n    }\n\n    debugCall(`response reasoning content: ${accumulatedReasoning}`);\n    debugCall(`response content: ${content}`);\n    assert(content, 'empty content');\n\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      } as OpenAI.CompletionUsage;\n    }\n\n    return {\n      content: content || '',\n      reasoning_content: accumulatedReasoning || undefined,\n      usage: buildUsageInfo(usage),\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service (${modelName}): ${e.message}\\nTrouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport async function callAIWithObjectResponse<T>(\n  messages: ChatCompletionMessageParam[],\n  modelConfig: IModelConfig,\n  options?: {\n    deepThink?: DeepThinkOption;\n  },\n): Promise<{\n  content: T;\n  contentString: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const response = await callAI(messages, modelConfig, {\n    deepThink: options?.deepThink,\n  });\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  assert(\n    typeof jsonContent === 'object',\n    `failed to parse json response from model (${modelConfig.modelName}): ${response.content}`,\n  );\n  return {\n    content: jsonContent,\n    contentString: response.content,\n    usage: response.usage,\n    reasoning_content: response.reasoning_content,\n  };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function resolveDeepThinkConfig({\n  deepThink,\n  vlMode,\n}: {\n  deepThink?: DeepThinkOption;\n  vlMode?: TVlModeTypes;\n}): {\n  config: Record<string, unknown>;\n  debugMessage?: string;\n  warningMessage?: string;\n} {\n  const normalizedDeepThink = deepThink === 'unset' ? undefined : deepThink;\n\n  if (normalizedDeepThink === undefined) {\n    return { config: {}, debugMessage: undefined };\n  }\n\n  if (vlMode === 'qwen3-vl') {\n    return {\n      config: { enable_thinking: normalizedDeepThink },\n      debugMessage: `deepThink mapped to enable_thinking=${normalizedDeepThink} for qwen3-vl`,\n    };\n  }\n\n  if (vlMode === 'doubao-vision') {\n    return {\n      config: {\n        thinking: { type: normalizedDeepThink ? 'enabled' : 'disabled' },\n      },\n      debugMessage: `deepThink mapped to thinking.type=${normalizedDeepThink ? 'enabled' : 'disabled'} for doubao-vision`,\n    };\n  }\n\n  if (vlMode === 'glm-v') {\n    return {\n      config: {\n        thinking: { type: normalizedDeepThink ? 'enabled' : 'disabled' },\n      },\n      debugMessage: `deepThink mapped to thinking.type=${normalizedDeepThink ? 'enabled' : 'disabled'} for glm-v`,\n    };\n  }\n\n  return {\n    config: {},\n    debugMessage: `deepThink ignored: unsupported model_family \"${vlMode ?? 'default'}\"`,\n    warningMessage: `The \"deepThink\" option is not supported for model_family \"${vlMode ?? 'default'}\".`,\n  };\n}\n\n/**\n * Normalize a parsed JSON object by trimming whitespace from:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction normalizeJsonObject(obj: any): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeJsonObject(item));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n\n      // Recursively normalize the value\n      let normalizedValue = normalizeJsonObject(value);\n\n      // Trim all string values\n      if (typeof normalizedValue === 'string') {\n        normalizedValue = normalizedValue.trim();\n      }\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n\n  let parsed: any;\n  let lastError: unknown;\n  try {\n    parsed = JSON.parse(cleanJsonString);\n    return normalizeJsonObject(parsed);\n  } catch (error) {\n    lastError = error;\n  }\n  try {\n    parsed = JSON.parse(jsonrepair(cleanJsonString));\n    return normalizeJsonObject(parsed);\n  } catch (error) {\n    lastError = error;\n  }\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    try {\n      parsed = JSON.parse(jsonrepair(jsonString));\n      return normalizeJsonObject(parsed);\n    } catch (error) {\n      lastError = error;\n    }\n  }\n  throw Error(\n    `failed to parse LLM response into JSON. Error - ${String(\n      lastError ?? 'unknown error',\n    )}. Response - \\n ${input}`,\n  );\n}\n"],"names":["createChatClient","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","modelDescription","uiTarsVersion","vlMode","createOpenAIClient","timeout","proxyAgent","debugProxy","getDebug","sanitizeProxyUrl","url","parsed","URL","ifInBrowser","console","moduleName","ProxyAgent","socksDispatcher","proxyUrl","Error","port","Number","protocol","socksType","decodeURIComponent","error","openAIOptions","baseOpenAI","OpenAI","openai","globalConfigManager","MIDSCENE_LANGSMITH_DEBUG","langsmithModule","wrapOpenAI","MIDSCENE_LANGFUSE_DEBUG","langfuseModule","observeOpenAI","wrappedClient","callAI","messages","options","completion","maxTokens","MIDSCENE_MODEL_MAX_TOKENS","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","temperature","isStreaming","content","accumulated","accumulatedReasoning","usage","timeCost","buildUsageInfo","usageData","cachedInputTokens","commonConfig","deepThinkConfig","debugMessage","warningMessage","resolveDeepThinkConfig","stream","chunk","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","result","JSON","assert","e","newError","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","deepThink","normalizedDeepThink","normalizeJsonObject","obj","Array","item","normalized","key","value","Object","trimmedKey","normalizedValue","cleanJsonString","lastError","jsonrepair","jsonString","String"],"mappings":";;;;;AAyBA,eAAeA,iBAAiB,EAC9BC,WAAW,EAGZ;IAOC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACNC,kBAAkB,EAClBC,OAAO,EACR,GAAGX;IAEJ,IAAIY;IACJ,MAAMC,aAAaC,SAAS;IAI5B,MAAMC,mBAAmB,CAACC;QACxB,IAAI;YACF,MAAMC,SAAS,IAAIC,IAAIF;YACvB,IAAIC,OAAO,QAAQ,EAAE;gBAEnBA,OAAO,QAAQ,GAAG;gBAClB,OAAOA,OAAO,IAAI;YACpB;YACA,OAAOD;QACT,EAAE,OAAM;YAEN,OAAOA;QACT;IACF;IAEA,IAAId,WAAW;QACbW,WAAW,oBAAoBE,iBAAiBb;QAChD,IAAIiB,aACFC,QAAQ,IAAI,CACV;aAEG;YAEL,MAAMC,aAAa;YACnB,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAACD;YACpCT,aAAa,IAAIU,WAAW;gBAC1B,KAAKpB;YAEP;QACF;IACF,OAAO,IAAID,YAAY;QACrBY,WAAW,qBAAqBE,iBAAiBd;QACjD,IAAIkB,aACFC,QAAQ,IAAI,CACV;aAGF,IAAI;YAEF,MAAMC,aAAa;YACnB,MAAM,EAAEE,eAAe,EAAE,GAAG,MAAM,MAAM,CAACF;YAEzC,MAAMG,WAAW,IAAIN,IAAIjB;YAGzB,IAAI,CAACuB,SAAS,QAAQ,EACpB,MAAM,IAAIC,MAAM;YAIlB,MAAMC,OAAOC,OAAO,QAAQ,CAACH,SAAS,IAAI,EAAE;YAC5C,IAAI,CAACA,SAAS,IAAI,IAAIG,OAAO,KAAK,CAACD,OACjC,MAAM,IAAID,MAAM;YAIlB,MAAMG,WAAWJ,SAAS,QAAQ,CAAC,OAAO,CAAC,KAAK;YAChD,MAAMK,YACJD,AAAa,aAAbA,WAAwB,IAAIA,AAAa,aAAbA,WAAwB,IAAI;YAE1DhB,aAAaW,gBAAgB;gBAC3B,MAAMM;gBACN,MAAML,SAAS,QAAQ;gBACvBE;gBACA,GAAIF,SAAS,QAAQ,GACjB;oBACE,QAAQM,mBAAmBN,SAAS,QAAQ;oBAC5C,UAAUM,mBAAmBN,SAAS,QAAQ,IAAI;gBACpD,IACA,CAAC,CAAC;YACR;YACAX,WAAW,uCAAuC;gBAChD,MAAMgB;gBACN,MAAML,SAAS,QAAQ;gBACvB,MAAME;YACR;QACF,EAAE,OAAOK,OAAO;YACdX,QAAQ,KAAK,CAAC,oCAAoCW;YAClD,MAAM,IAAIN,MACR,CAAC,yBAAyB,EAAExB,WAAW,+GAA+G,CAAC;QAE3J;IAEJ;IAEA,MAAM+B,gBAAgB;QACpB,SAAS5B;QACT,QAAQC;QAGR,GAAIO,aAAa;YAAE,cAAc;gBAAE,YAAYA;YAAkB;QAAE,IAAI,CAAC,CAAC;QACzE,GAAGN,iBAAiB;QACpB,GAAI,AAAmB,YAAnB,OAAOK,UAAuB;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAClD,yBAAyB;IAC3B;IAEA,MAAMsB,aAAa,IAAIC,SAAOF;IAE9B,IAAIG,SAAiBF;IAGrB,IACEE,UACAC,oBAAoB,qBAAqB,CAACC,2BAC1C;QACA,IAAIlB,aACF,MAAM,IAAIM,MAAM;QAElBL,QAAQ,GAAG,CAAC;QAEZ,MAAMkB,kBAAkB;QACxB,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAACD;QACpCH,SAASI,WAAWJ;IACtB;IAGA,IACEA,UACAC,oBAAoB,qBAAqB,CAACI,0BAC1C;QACA,IAAIrB,aACF,MAAM,IAAIM,MAAM;QAElBL,QAAQ,GAAG,CAAC;QAEZ,MAAMqB,iBAAiB;QACvB,MAAM,EAAEC,aAAa,EAAE,GAAG,MAAM,MAAM,CAACD;QACvCN,SAASO,cAAcP;IACzB;IAEA,IAAIzB,oBAAoB;QACtB,MAAMiC,gBAAgB,MAAMjC,mBAAmBuB,YAAYD;QAE3D,IAAIW,eACFR,SAASQ;IAEb;IAEA,OAAO;QACL,YAAYR,OAAO,IAAI,CAAC,WAAW;QACnChC;QACAI;QACAC;QACAC;IACF;AACF;AAEO,eAAemC,OACpBC,QAAsC,EACtC7C,WAAyB,EACzB8C,OAIC;IAOD,MAAM,EAAEC,UAAU,EAAE5C,SAAS,EAAEI,gBAAgB,EAAEC,aAAa,EAAEC,MAAM,EAAE,GACtE,MAAMV,iBAAiB;QACrBC;IACF;IAEF,MAAMgD,YACJZ,oBAAoB,yBAAyB,CAACa,8BAC9Cb,oBAAoB,yBAAyB,CAACc;IAChD,MAAMC,YAAYrC,SAAS;IAC3B,MAAMsC,oBAAoBtC,SAAS;IACnC,MAAMuC,qBAAqBvC,SAAS;IAEpC,MAAMwC,YAAYC,KAAK,GAAG;IAC1B,MAAMC,cAAcxD,YAAY,WAAW,IAAI;IAE/C,MAAMyD,cAAcX,SAAS,UAAUA,SAAS;IAChD,IAAIY;IACJ,IAAIC,cAAc;IAClB,IAAIC,uBAAuB;IAC3B,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,iBAAiB,CAACC;QACtB,IAAI,CAACA,WAAW;QAEhB,MAAMC,oBACJD,WACC,uBAAuB;QAE1B,OAAO;YACL,eAAeA,UAAU,aAAa,IAAI;YAC1C,mBAAmBA,UAAU,iBAAiB,IAAI;YAClD,cAAcA,UAAU,YAAY,IAAI;YACxC,cAAcC,qBAAqB;YACnC,WAAWH,YAAY;YACvB,YAAY3D;YACZ,mBAAmBI;YACnB,QAAQP,YAAY,MAAM;QAC5B;IACF;IAEA,MAAMkE,eAAe;QACnBV;QACA,QAAQ,CAAC,CAACC;QACV,YAAYT;QACZ,GAAIvC,AAAW,iBAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IACA,MAAM,EACJ,QAAQ0D,eAAe,EACvBC,YAAY,EACZC,cAAc,EACf,GAAGC,uBAAuB;QACzB,WAAWxB,SAAS;QACpBrC;IACF;IACA,IAAI2D,cACFjB,UAAUiB;IAEZ,IAAIC,gBAAgB;QAClBlB,UAAUkB;QACVjD,QAAQ,IAAI,CAACiD;IACf;IAEA,IAAI;QACFlB,UACE,CAAC,QAAQ,EAAEM,cAAc,eAAe,GAAG,WAAW,EAAEtD,WAAW;QAGrE,IAAIsD,aAAa;YACf,MAAMc,SAAU,MAAMxB,WAAW,MAAM,CACrC;gBACE,OAAO5C;gBACP0C;gBACA,GAAGqB,YAAY;gBACf,GAAGC,eAAe;YACpB,GACA;gBACE,QAAQ;YACV;YAKF,WAAW,MAAMK,SAASD,OAAQ;gBAChC,MAAMb,UAAUc,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,OAAO,WAAW;gBACtD,MAAMC,oBACHD,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,OAAe,qBAAqB;gBAG3D,IAAIA,MAAM,KAAK,EACbX,QAAQW,MAAM,KAAK;gBAGrB,IAAId,WAAWe,mBAAmB;oBAChCd,eAAeD;oBACfE,wBAAwBa;oBACxB,MAAMC,YAAiC;wBACrChB;wBACAe;wBACAd;wBACA,YAAY;wBACZ,OAAOgB;oBACT;oBACA7B,QAAQ,OAAO,CAAE4B;gBACnB;gBAGA,IAAIF,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,eAAe;oBACrCV,WAAWP,KAAK,GAAG,KAAKD;oBAGxB,IAAI,CAACO,OAAO;wBAEV,MAAMe,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAClB,YAAY,MAAM,GAAG;wBAElCE,QAAQ;4BACN,eAAee;4BACf,mBAAmBA;4BACnB,cAAcA,AAAkB,IAAlBA;wBAChB;oBACF;oBAGA,MAAME,aAAkC;wBACtC,SAAS;wBACTnB;wBACA,mBAAmB;wBACnB,YAAY;wBACZ,OAAOI,eAAeF;oBACxB;oBACAf,QAAQ,OAAO,CAAEgC;oBACjB;gBACF;YACF;YACApB,UAAUC;YACVP,kBACE,CAAC,iBAAiB,EAAEjD,UAAU,QAAQ,EAAEM,UAAU,UAAU,WAAW,EAAEqD,SAAS,eAAe,EAAEN,eAAe,IAAI;QAE1H,OAAO;YACL,MAAMuB,SAAS,MAAMhC,WAAW,MAAM,CAAC;gBACrC,OAAO5C;gBACP0C;gBACA,GAAGqB,YAAY;gBACf,GAAGC,eAAe;YACpB;YACAL,WAAWP,KAAK,GAAG,KAAKD;YAExBF,kBACE,CAAC,OAAO,EAAEjD,UAAU,QAAQ,EAAEM,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAEuE,OAAO,KAAK,EAAE,iBAAiB,GAAG,qBAAqB,EAAEA,OAAO,KAAK,EAAE,qBAAqB,GAAG,gBAAgB,EAAEA,OAAO,KAAK,EAAE,gBAAgB,GAAG,WAAW,EAAEjB,SAAS,aAAa,EAAEiB,OAAO,WAAW,IAAI,GAAG,eAAe,EAAEvB,eAAe,IAAI;YAG9VH,mBAAmB,CAAC,oBAAoB,EAAE2B,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;YAExEE,OACEF,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;YAEhErB,UAAUqB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;YAC3CnB,uBACGmB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,EAAU,qBAAqB;YAC3DlB,QAAQkB,OAAO,KAAK;QACtB;QAEA5B,UAAU,CAAC,4BAA4B,EAAES,sBAAsB;QAC/DT,UAAU,CAAC,kBAAkB,EAAEO,SAAS;QACxCuB,OAAOvB,SAAS;QAGhB,IAAID,eAAe,CAACI,OAAO;YAEzB,MAAMe,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEnB,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCG,QAAQ;gBACN,eAAee;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASlB,WAAW;YACpB,mBAAmBE,wBAAwBe;YAC3C,OAAOZ,eAAeF;YACtB,YAAY,CAAC,CAACJ;QAChB;IACF,EAAE,OAAOyB,GAAQ;QACf9D,QAAQ,KAAK,CAAC,kBAAkB8D;QAChC,MAAMC,WAAW,IAAI1D,MACnB,CAAC,eAAe,EAAEgC,cAAc,eAAe,GAAG,kBAAkB,EAAEtD,UAAU,GAAG,EAAE+E,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC9J;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,eAAeC,yBACpBvC,QAAsC,EACtC7C,WAAyB,EACzB8C,OAEC;IAOD,MAAMuC,WAAW,MAAMzC,OAAOC,UAAU7C,aAAa;QACnD,WAAW8C,SAAS;IACtB;IACAmC,OAAOI,UAAU;IACjB,MAAM5E,SAAST,YAAY,MAAM;IACjC,MAAMsF,cAAcC,cAAcF,SAAS,OAAO,EAAE5E;IACpDwE,OACE,AAAuB,YAAvB,OAAOK,aACP,CAAC,0CAA0C,EAAEtF,YAAY,SAAS,CAAC,GAAG,EAAEqF,SAAS,OAAO,EAAE;IAE5F,OAAO;QACL,SAASC;QACT,eAAeD,SAAS,OAAO;QAC/B,OAAOA,SAAS,KAAK;QACrB,mBAAmBA,SAAS,iBAAiB;IAC/C;AACF;AAEO,eAAeG,yBACpBC,IAAY,EACZzF,WAAyB;IAEzB,MAAM,EAAE0D,OAAO,EAAEG,KAAK,EAAE,GAAG,MAAMjB,OAAO6C,MAAMzF;IAC9C,OAAO;QAAE0D;QAASG;IAAM;AAC1B;AAEO,SAAS6B,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASzB,uBAAuB,EACrC0B,SAAS,EACTvF,MAAM,EAIP;IAKC,MAAMwF,sBAAsBD,AAAc,YAAdA,YAAwBrB,SAAYqB;IAEhE,IAAIC,AAAwBtB,WAAxBsB,qBACF,OAAO;QAAE,QAAQ,CAAC;QAAG,cAActB;IAAU;IAG/C,IAAIlE,AAAW,eAAXA,QACF,OAAO;QACL,QAAQ;YAAE,iBAAiBwF;QAAoB;QAC/C,cAAc,CAAC,oCAAoC,EAAEA,oBAAoB,aAAa,CAAC;IACzF;IAGF,IAAIxF,AAAW,oBAAXA,QACF,OAAO;QACL,QAAQ;YACN,UAAU;gBAAE,MAAMwF,sBAAsB,YAAY;YAAW;QACjE;QACA,cAAc,CAAC,kCAAkC,EAAEA,sBAAsB,YAAY,WAAW,kBAAkB,CAAC;IACrH;IAGF,IAAIxF,AAAW,YAAXA,QACF,OAAO;QACL,QAAQ;YACN,UAAU;gBAAE,MAAMwF,sBAAsB,YAAY;YAAW;QACjE;QACA,cAAc,CAAC,kCAAkC,EAAEA,sBAAsB,YAAY,WAAW,UAAU,CAAC;IAC7G;IAGF,OAAO;QACL,QAAQ,CAAC;QACT,cAAc,CAAC,6CAA6C,EAAExF,UAAU,UAAU,CAAC,CAAC;QACpF,gBAAgB,CAAC,0DAA0D,EAAEA,UAAU,UAAU,EAAE,CAAC;IACtG;AACF;AAQA,SAASyF,oBAAoBC,GAAQ;IAEnC,IAAIA,QAAAA,KACF,OAAOA;IAIT,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACE,OAASH,oBAAoBG;IAI/C,IAAI,AAAe,YAAf,OAAOF,KAAkB;QAC3B,MAAMG,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACN,KAAM;YAE9C,MAAMO,aAAaH,IAAI,IAAI;YAG3B,IAAII,kBAAkBT,oBAAoBM;YAG1C,IAAI,AAA2B,YAA3B,OAAOG,iBACTA,kBAAkBA,gBAAgB,IAAI;YAGxCL,UAAU,CAACI,WAAW,GAAGC;QAC3B;QAEA,OAAOL;IACT;IAGA,IAAI,AAAe,YAAf,OAAOH,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEO,SAASZ,cAAcQ,KAAa,EAAEtF,MAAgC;IAC3E,MAAMmG,kBAAkBlB,yBAAyBK;IAEjD,IAAIa,iBAAiB,MAAM,oBACzB,OAAOA,gBACJ,KAAK,CAAC,oBACL,MAAM,GACP,IAAIjF;IAGT,IAAIV;IACJ,IAAI4F;IACJ,IAAI;QACF5F,SAAS+D,KAAK,KAAK,CAAC4B;QACpB,OAAOV,oBAAoBjF;IAC7B,EAAE,OAAOc,OAAO;QACd8E,YAAY9E;IACd;IACA,IAAI;QACFd,SAAS+D,KAAK,KAAK,CAAC8B,WAAWF;QAC/B,OAAOV,oBAAoBjF;IAC7B,EAAE,OAAOc,OAAO;QACd8E,YAAY9E;IACd;IAEA,IAAItB,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAMsG,aAAajB,yBAAyBc;QAC5C,IAAI;YACF3F,SAAS+D,KAAK,KAAK,CAAC8B,WAAWC;YAC/B,OAAOb,oBAAoBjF;QAC7B,EAAE,OAAOc,OAAO;YACd8E,YAAY9E;QACd;IACF;IACA,MAAMN,MACJ,CAAC,gDAAgD,EAAEuF,OACjDH,aAAa,iBACb,gBAAgB,EAAEd,OAAO;AAE/B"}

package/dist/es/ai-model/ui-tars-planning.mjs

@@ -4,7 +4,6 @@ import { getDebug } from "@midscene/shared/logger";
 import { transformHotkeyInput } from "@midscene/shared/us-keyboard-layout";
 import { assert } from "@midscene/shared/utils";
 import { actionParser } from "@ui-tars/action-parser";
-import { LatestLocateRecorder } from "./latest-locate-recorder.mjs";
 import { getSummary, getUiTarsPlanningPrompt } from "./prompt/ui-tars-planning.mjs";
 import { callAIWithStringResponse } from "./service-caller/index.mjs";
 const debug = getDebug('ui-tars-planning');
@@ -15,7 +14,6 @@ const pointToBbox = (point, width, height)=>[
         Math.round(Math.min(point.x + bboxSize / 2, width)),
         Math.round(Math.min(point.y + bboxSize / 2, height))
     ];
-const lastLocateRecorder = new LatestLocateRecorder();
 async function uiTarsPlanning(userInstruction, options) {
     const { conversationHistory, context, modelConfig, actionContext } = options;
     const { uiTarsModelVersion } = modelConfig;
@@ -64,53 +62,47 @@ async function uiTarsPlanning(userInstruction, options) {
         if ('click' === actionType) {
             assert(action.action_inputs.start_box, 'start_box is required');
             const point = getPoint(action.action_inputs.start_box, size);
-            const locate = {
-                prompt: action.thought || '',
-                bbox: pointToBbox({
-                    x: point[0],
-                    y: point[1]
-                }, size.width, size.height)
-            };
-            lastLocateRecorder.recordLocate(locate, 'click');
             transformActions.push({
                 type: 'Tap',
                 param: {
-                    locate: locate
+                    locate: {
+                        prompt: action.thought || '',
+                        bbox: pointToBbox({
+                            x: point[0],
+                            y: point[1]
+                        }, size.width, size.height)
+                    }
                 }
             });
         } else if ('left_double' === actionType) {
             assert(action.action_inputs.start_box, 'start_box is required');
             const point = getPoint(action.action_inputs.start_box, size);
-            const locate = {
-                prompt: action.thought || '',
-                bbox: pointToBbox({
-                    x: point[0],
-                    y: point[1]
-                }, size.width, size.height)
-            };
-            lastLocateRecorder.recordLocate(locate, 'left_double');
             transformActions.push({
                 type: 'DoubleClick',
                 param: {
-                    locate: locate
+                    locate: {
+                        prompt: action.thought || '',
+                        bbox: pointToBbox({
+                            x: point[0],
+                            y: point[1]
+                        }, size.width, size.height)
+                    }
                 },
                 thought: action.thought || ''
             });
         } else if ('right_single' === actionType) {
             assert(action.action_inputs.start_box, 'start_box is required');
             const point = getPoint(action.action_inputs.start_box, size);
-            const locate = {
-                prompt: action.thought || '',
-                bbox: pointToBbox({
-                    x: point[0],
-                    y: point[1]
-                }, size.width, size.height)
-            };
-            lastLocateRecorder.recordLocate(locate, 'right_single');
             transformActions.push({
                 type: 'RightClick',
                 param: {
-                    locate: locate
+                    locate: {
+                        prompt: action.thought || '',
+                        bbox: pointToBbox({
+                            x: point[0],
+                            y: point[1]
+                        }, size.width, size.height)
+                    }
                 },
                 thought: action.thought || ''
             });
@@ -139,18 +131,14 @@ async function uiTarsPlanning(userInstruction, options) {
                 },
                 thought: action.thought || ''
             });
-        } else if ('type' === actionType) {
-            const { locate: latestLocate, source } = lastLocateRecorder.getLatestLocate();
-            debug(`use latestLocate from ${source} as locate when Input`, latestLocate);
-            transformActions.push({
-                type: 'Input',
-                param: {
-                    value: action.action_inputs.content,
-                    locate: latestLocate
-                },
-                thought: action.thought || ''
-            });
-        } else if ('scroll' === actionType) transformActions.push({
+        } else if ('type' === actionType) transformActions.push({
+            type: 'Input',
+            param: {
+                value: action.action_inputs.content
+            },
+            thought: action.thought || ''
+        });
+        else if ('scroll' === actionType) transformActions.push({
             type: 'Scroll',
             param: {
                 direction: action.action_inputs.direction