@donggui/core 1.5.4-donggui.3

package/dist/es/ai-model/inspect.mjs

@@ -0,0 +1,386 @@
+import { generateElementByPoint, generateElementByRect } from "@midscene/shared/extractor/dom-util";
+import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl, scaleImage } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+import { assert } from "@midscene/shared/utils";
+import { adaptBboxToRect, expandSearchArea, mergeRects } from "../common.mjs";
+import { parseAutoGLMLocateResponse } from "./auto-glm/parser.mjs";
+import { getAutoGLMLocatePrompt } from "./auto-glm/prompt.mjs";
+import { isAutoGLM } from "./auto-glm/util.mjs";
+import { extractDataQueryPrompt, parseXMLExtractionResponse, systemPromptToExtract } from "./prompt/extraction.mjs";
+import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
+import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
+import { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive } from "./prompt/order-sensitive-judge.mjs";
+import { AIResponseParseError, callAI, callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
+const debugInspect = getDebug('ai:inspect');
+const debugSection = getDebug('ai:section');
+const extraTextFromUserPrompt = (prompt)=>{
+    if ('string' == typeof prompt) return prompt;
+    return prompt.prompt;
+};
+const promptsToChatParam = async (multimodalPrompt)=>{
+    const msgs = [];
+    if (multimodalPrompt?.images?.length) {
+        msgs.push({
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: 'Next, I will provide all the reference images.'
+                }
+            ]
+        });
+        for (const item of multimodalPrompt.images){
+            const base64 = await preProcessImageUrl(item.url, !!multimodalPrompt.convertHttpImage2Base64);
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'text',
+                        text: `this is the reference image named '${item.name}':`
+                    }
+                ]
+            });
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: base64,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            });
+        }
+    }
+    return msgs;
+};
+async function AiLocateElement(options) {
+    const { context, targetElementDescription, modelConfig } = options;
+    const { modelFamily } = modelConfig;
+    const screenshotBase64 = context.screenshot.base64;
+    assert(targetElementDescription, "cannot find the target element description");
+    const targetElementDescriptionText = extraTextFromUserPrompt(targetElementDescription);
+    const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);
+    const systemPrompt = isAutoGLM(modelFamily) ? getAutoGLMLocatePrompt(modelFamily) : systemPromptToLocateElement(modelFamily);
+    let imagePayload = screenshotBase64;
+    let imageWidth = context.shotSize.width;
+    let imageHeight = context.shotSize.height;
+    let originalImageWidth = imageWidth;
+    let originalImageHeight = imageHeight;
+    if (options.searchConfig) {
+        assert(options.searchConfig.rect, 'searchArea is provided but its rect cannot be found. Failed to locate element');
+        assert(options.searchConfig.imageBase64, 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element');
+        imagePayload = options.searchConfig.imageBase64;
+        imageWidth = options.searchConfig.rect?.width;
+        imageHeight = options.searchConfig.rect?.height;
+        originalImageWidth = imageWidth;
+        originalImageHeight = imageHeight;
+    } else if ('qwen2.5-vl' === modelFamily) {
+        const paddedResult = await paddingToMatchBlockByBase64(imagePayload);
+        imageWidth = paddedResult.width;
+        imageHeight = paddedResult.height;
+        imagePayload = paddedResult.imageBase64;
+    }
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: isAutoGLM(modelFamily) ? `Tap: ${userInstructionPrompt}` : userInstructionPrompt
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof targetElementDescription) {
+        const addOns = await promptsToChatParam({
+            images: targetElementDescription.images,
+            convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    if (isAutoGLM(modelFamily)) {
+        const { content: rawResponseContent, usage } = await callAIWithStringResponse(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+        debugInspect('auto-glm rawResponse:', rawResponseContent);
+        const parsed = parseAutoGLMLocateResponse(rawResponseContent);
+        debugInspect('auto-glm thinking:', parsed.think);
+        debugInspect('auto-glm coordinates:', parsed.coordinates);
+        let resRect;
+        let matchedElements = [];
+        let errors = [];
+        if (parsed.error || !parsed.coordinates) {
+            errors = [
+                parsed.error || 'Failed to parse auto-glm response'
+            ];
+            debugInspect('auto-glm parse error:', errors[0]);
+        } else {
+            const { x, y } = parsed.coordinates;
+            debugInspect('auto-glm coordinates [0-999]:', {
+                x,
+                y
+            });
+            const pixelX = Math.round(x * imageWidth / 1000);
+            const pixelY = Math.round(y * imageHeight / 1000);
+            debugInspect('auto-glm pixel coordinates:', {
+                pixelX,
+                pixelY
+            });
+            let finalX = pixelX;
+            let finalY = pixelY;
+            if (options.searchConfig?.rect) {
+                finalX += options.searchConfig.rect.left;
+                finalY += options.searchConfig.rect.top;
+            }
+            const element = generateElementByPoint([
+                finalX,
+                finalY
+            ], targetElementDescriptionText);
+            resRect = element.rect;
+            debugInspect('auto-glm resRect:', resRect);
+            if (element) matchedElements = [
+                element
+            ];
+        }
+        return {
+            rect: resRect,
+            parseResult: {
+                elements: matchedElements,
+                errors
+            },
+            rawResponse: rawResponseContent,
+            usage,
+            reasoning_content: parsed.think
+        };
+    }
+    let res;
+    try {
+        res = await callAIWithObjectResponse(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+    } catch (callError) {
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        const rawResponse = callError instanceof AIResponseParseError ? callError.rawResponse : errorMessage;
+        const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
+        return {
+            rect: void 0,
+            parseResult: {
+                elements: [],
+                errors: [
+                    `AI call error: ${errorMessage}`
+                ]
+            },
+            rawResponse,
+            usage,
+            reasoning_content: void 0
+        };
+    }
+    const rawResponse = JSON.stringify(res.content);
+    let resRect;
+    let matchedElements = [];
+    let errors = 'errors' in res.content ? res.content.errors : [];
+    try {
+        if ('bbox' in res.content && Array.isArray(res.content.bbox) && res.content.bbox.length >= 1) {
+            resRect = adaptBboxToRect(res.content.bbox, imageWidth, imageHeight, options.searchConfig?.rect?.left, options.searchConfig?.rect?.top, originalImageWidth, originalImageHeight, modelFamily, options.searchConfig?.scale);
+            debugInspect('resRect', resRect);
+            const element = generateElementByRect(resRect, targetElementDescriptionText);
+            errors = [];
+            if (element) matchedElements = [
+                element
+            ];
+        }
+    } catch (e) {
+        const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
+        if (errors && errors?.length !== 0) errors.push(`(${msg})`);
+        else errors = [
+            msg
+        ];
+    }
+    return {
+        rect: resRect,
+        parseResult: {
+            elements: matchedElements,
+            errors: errors
+        },
+        rawResponse,
+        usage: res.usage,
+        reasoning_content: res.reasoning_content
+    };
+}
+async function AiLocateSection(options) {
+    const { context, sectionDescription, modelConfig } = options;
+    const { modelFamily } = modelConfig;
+    const screenshotBase64 = context.screenshot.base64;
+    const systemPrompt = systemPromptToLocateSection(modelFamily);
+    const sectionLocatorInstructionText = sectionLocatorInstruction(extraTextFromUserPrompt(sectionDescription));
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: screenshotBase64,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: sectionLocatorInstructionText
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof sectionDescription) {
+        const addOns = await promptsToChatParam({
+            images: sectionDescription.images,
+            convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    let result;
+    try {
+        result = await callAIWithObjectResponse(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+    } catch (callError) {
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        const rawResponse = callError instanceof AIResponseParseError ? callError.rawResponse : errorMessage;
+        const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
+        return {
+            rect: void 0,
+            imageBase64: void 0,
+            error: `AI call error: ${errorMessage}`,
+            rawResponse,
+            usage
+        };
+    }
+    let sectionRect;
+    const sectionBbox = result.content.bbox;
+    if (sectionBbox) {
+        const targetRect = adaptBboxToRect(sectionBbox, context.shotSize.width, context.shotSize.height, 0, 0, context.shotSize.width, context.shotSize.height, modelFamily);
+        debugSection('original targetRect %j', targetRect);
+        const referenceBboxList = result.content.references_bbox || [];
+        debugSection('referenceBboxList %j', referenceBboxList);
+        const referenceRects = referenceBboxList.filter((bbox)=>Array.isArray(bbox)).map((bbox)=>adaptBboxToRect(bbox, context.shotSize.width, context.shotSize.height, 0, 0, context.shotSize.width, context.shotSize.height, modelFamily));
+        debugSection('referenceRects %j', referenceRects);
+        const mergedRect = mergeRects([
+            targetRect,
+            ...referenceRects
+        ]);
+        debugSection('mergedRect %j', mergedRect);
+        sectionRect = expandSearchArea(mergedRect, context.shotSize);
+        debugSection('expanded sectionRect %j', sectionRect);
+    }
+    let imageBase64 = screenshotBase64;
+    let scale;
+    if (sectionRect) {
+        const originalWidth = sectionRect.width;
+        const originalHeight = sectionRect.height;
+        const croppedResult = await cropByRect(screenshotBase64, sectionRect, 'qwen2.5-vl' === modelFamily);
+        const scaleRatio = 2;
+        const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);
+        imageBase64 = scaledResult.imageBase64;
+        scale = scaleRatio;
+        sectionRect.width = scaledResult.width;
+        sectionRect.height = scaledResult.height;
+        debugSection('scaled sectionRect from %dx%d to %dx%d (scale=%d)', originalWidth, originalHeight, sectionRect.width, sectionRect.height, scale);
+    }
+    return {
+        rect: sectionRect,
+        imageBase64,
+        scale,
+        error: result.content.error,
+        rawResponse: JSON.stringify(result.content),
+        usage: result.usage
+    };
+}
+async function AiExtractElementInfo(options) {
+    const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } = options;
+    const systemPrompt = systemPromptToExtract();
+    const screenshotBase64 = context.screenshot.base64;
+    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery);
+    const userContent = [];
+    if (extractOption?.screenshotIncluded !== false) userContent.push({
+        type: 'image_url',
+        image_url: {
+            url: screenshotBase64,
+            detail: 'high'
+        }
+    });
+    userContent.push({
+        type: 'text',
+        text: extractDataPromptText
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userContent
+        }
+    ];
+    if (multimodalPrompt) {
+        const addOns = await promptsToChatParam({
+            images: multimodalPrompt.images,
+            convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const { content: rawResponse, usage, reasoning_content } = await callAI(msgs, modelConfig);
+    let parseResult;
+    try {
+        parseResult = parseXMLExtractionResponse(rawResponse);
+    } catch (parseError) {
+        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
+        throw new AIResponseParseError(`XML parse error: ${errorMessage}`, rawResponse, usage);
+    }
+    return {
+        parseResult,
+        rawResponse,
+        usage,
+        reasoning_content
+    };
+}
+async function AiJudgeOrderSensitive(description, callAIFn, modelConfig) {
+    const systemPrompt = systemPromptToJudgeOrderSensitive();
+    const userPrompt = orderSensitiveJudgePrompt(description);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userPrompt
+        }
+    ];
+    const result = await callAIFn(msgs, modelConfig);
+    return {
+        isOrderSensitive: result.content.isOrderSensitive ?? false,
+        usage: result.usage
+    };
+}
+export { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection };
+
+//# sourceMappingURL=inspect.mjs.map

package/dist/es/ai-model/inspect.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  generateElementByPoint,\n  generateElementByRect,\n} from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n  scaleImage,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { adaptBboxToRect, expandSearchArea, mergeRects } from '../common';\nimport { parseAutoGLMLocateResponse } from './auto-glm/parser';\nimport { getAutoGLMLocatePrompt } from './auto-glm/prompt';\nimport { isAutoGLM } from './auto-glm/util';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n  callAIWithStringResponse,\n} from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}':`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const { context, targetElementDescription, modelConfig } = options;\n  const { modelFamily } = modelConfig;\n  const screenshotBase64 = context.screenshot.base64;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const targetElementDescriptionText = extraTextFromUserPrompt(\n    targetElementDescription,\n  );\n  const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n  const systemPrompt = isAutoGLM(modelFamily)\n    ? getAutoGLMLocatePrompt(modelFamily)\n    : systemPromptToLocateElement(modelFamily);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.shotSize.width;\n  let imageHeight = context.shotSize.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (modelFamily === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: isAutoGLM(modelFamily)\n            ? `Tap: ${userInstructionPrompt}`\n            : userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  if (isAutoGLM(modelFamily)) {\n    const { content: rawResponseContent, usage } =\n      await callAIWithStringResponse(msgs, modelConfig, {\n        abortSignal: options.abortSignal,\n      });\n\n    debugInspect('auto-glm rawResponse:', rawResponseContent);\n\n    const parsed = parseAutoGLMLocateResponse(rawResponseContent);\n\n    debugInspect('auto-glm thinking:', parsed.think);\n    debugInspect('auto-glm coordinates:', parsed.coordinates);\n\n    let resRect: Rect | undefined;\n    let matchedElements: LocateResultElement[] = [];\n    let errors: string[] = [];\n\n    if (parsed.error || !parsed.coordinates) {\n      errors = [parsed.error || 'Failed to parse auto-glm response'];\n      debugInspect('auto-glm parse error:', errors[0]);\n    } else {\n      const { x, y } = parsed.coordinates;\n\n      debugInspect('auto-glm coordinates [0-999]:', { x, y });\n\n      // Convert auto-glm coordinates [0,999] to pixel bbox\n      // Map from [0,999] to pixel coordinates\n      const pixelX = Math.round((x * imageWidth) / 1000);\n      const pixelY = Math.round((y * imageHeight) / 1000);\n\n      debugInspect('auto-glm pixel coordinates:', { pixelX, pixelY });\n\n      // Apply offset if searching in a cropped area\n      let finalX = pixelX;\n      let finalY = pixelY;\n      if (options.searchConfig?.rect) {\n        finalX += options.searchConfig.rect.left;\n        finalY += options.searchConfig.rect.top;\n      }\n\n      const element: LocateResultElement = generateElementByPoint(\n        [finalX, finalY],\n        targetElementDescriptionText as string,\n      );\n\n      resRect = element.rect;\n      debugInspect('auto-glm resRect:', resRect);\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n\n    return {\n      rect: resRect,\n      parseResult: {\n        elements: matchedElements,\n        errors,\n      },\n      rawResponse: rawResponseContent,\n      usage,\n      reasoning_content: parsed.think,\n    };\n  }\n\n  let res: Awaited<\n    ReturnType<\n      typeof callAIWithObjectResponse<AIElementResponse | [number, number]>\n    >\n  >;\n  try {\n    res = await callAIWithObjectResponse<AIElementResponse | [number, number]>(\n      msgs,\n      modelConfig,\n      { abortSignal: options.abortSignal },\n    );\n  } catch (callError) {\n    // Return error with usage and rawResponse if available\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    return {\n      rect: undefined,\n      parseResult: {\n        elements: [],\n        errors: [`AI call error: ${errorMessage}`],\n      },\n      rawResponse,\n      usage,\n      reasoning_content: undefined,\n    };\n  }\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: LocateResultElement[] = [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if (\n      'bbox' in res.content &&\n      Array.isArray(res.content.bbox) &&\n      res.content.bbox.length >= 1\n    ) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        modelFamily,\n        options.searchConfig?.scale,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const element: LocateResultElement = generateElementByRect(\n        resRect,\n        targetElementDescriptionText as string,\n      );\n      errors = [];\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n    reasoning_content: res.reasoning_content,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  scale?: number;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { modelFamily } = modelConfig;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = systemPromptToLocateSection(modelFamily);\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  let result: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n  >;\n  try {\n    result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n      msgs,\n      modelConfig,\n      { abortSignal: options.abortSignal },\n    );\n  } catch (callError) {\n    // Return error with usage and rawResponse if available\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    return {\n      rect: undefined,\n      imageBase64: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse,\n      usage,\n    };\n  }\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.shotSize.width,\n      context.shotSize.height,\n      0,\n      0,\n      context.shotSize.width,\n      context.shotSize.height,\n      modelFamily,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.shotSize.width,\n          context.shotSize.height,\n          0,\n          0,\n          context.shotSize.width,\n          context.shotSize.height,\n          modelFamily,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    sectionRect = expandSearchArea(mergedRect, context.shotSize);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  let scale: number | undefined;\n\n  if (sectionRect) {\n    const originalWidth = sectionRect.width;\n    const originalHeight = sectionRect.height;\n\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      modelFamily === 'qwen2.5-vl',\n    );\n\n    const scaleRatio = 2;\n    const scaledResult = await scaleImage(\n      croppedResult.imageBase64,\n      scaleRatio,\n    );\n\n    imageBase64 = scaledResult.imageBase64;\n    scale = scaleRatio;\n    sectionRect.width = scaledResult.width;\n    sectionRect.height = scaledResult.height;\n\n    debugSection(\n      'scaled sectionRect from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      sectionRect.width,\n      sectionRect.height,\n      scale,\n    );\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    scale,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n  } = await callAI(msgs, modelConfig);\n\n  // Parse XML response to JSON object\n  let parseResult: AIDataExtractionResponse<T>;\n  try {\n    parseResult = parseXMLExtractionResponse<T>(rawResponse);\n  } catch (parseError) {\n    // Throw AIResponseParseError with usage and rawResponse preserved\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n    );\n  }\n\n  return {\n    parseResult,\n    rawResponse,\n    usage,\n    reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n  modelConfig: IModelConfig,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  const result = await callAIFn(msgs, modelConfig);\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","modelConfig","modelFamily","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","isAutoGLM","getAutoGLMLocatePrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","paddedResult","paddingToMatchBlockByBase64","addOns","rawResponseContent","usage","callAIWithStringResponse","parsed","parseAutoGLMLocateResponse","resRect","matchedElements","errors","x","y","pixelX","Math","pixelY","finalX","finalY","element","generateElementByPoint","res","callAIWithObjectResponse","callError","errorMessage","Error","String","rawResponse","AIResponseParseError","undefined","JSON","Array","adaptBboxToRect","generateElementByRect","e","msg","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","scale","originalWidth","originalHeight","croppedResult","cropByRect","scaleRatio","scaledResult","scaleImage","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","reasoning_content","callAI","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","callAIFn","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;AA6DA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OAMrC;IAUC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,WAAW,EAAE,GAAGH;IAC3D,MAAM,EAAEI,WAAW,EAAE,GAAGD;IACxB,MAAME,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElDK,OACEJ,0BACA;IAEF,MAAMK,+BAA+BhB,wBACnCW;IAEF,MAAMM,wBAAwBC,kBAAkBF;IAChD,MAAMG,eAAeC,UAAUP,eAC3BQ,uBAAuBR,eACvBS,4BAA4BT;IAEhC,IAAIU,eAAeT;IACnB,IAAIU,aAAad,QAAQ,QAAQ,CAAC,KAAK;IACvC,IAAIe,cAAcf,QAAQ,QAAQ,CAAC,MAAM;IACzC,IAAIgB,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIhB,QAAQ,YAAY,EAAE;QACxBM,OACEN,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFM,OACEN,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFc,eAAed,QAAQ,YAAY,CAAC,WAAW;QAC/Ce,aAAaf,QAAQ,YAAY,CAAC,IAAI,EAAE;QACxCgB,cAAchB,QAAQ,YAAY,CAAC,IAAI,EAAE;QACzCiB,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIZ,AAAgB,iBAAhBA,aAA8B;QACvC,MAAMe,eAAe,MAAMC,4BAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAMxB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASe;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMH,UAAUP,eACZ,CAAC,KAAK,EAAEI,uBAAuB,GAC/BA;gBACN;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAON,0BAAuC;QAChD,MAAMmB,SAAS,MAAM5B,mBAAmB;YACtC,QAAQS,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI0B;IACf;IAEA,IAAIV,UAAUP,cAAc;QAC1B,MAAM,EAAE,SAASkB,kBAAkB,EAAEC,KAAK,EAAE,GAC1C,MAAMC,yBAAyB7B,MAAMQ,aAAa;YAChD,aAAaH,QAAQ,WAAW;QAClC;QAEFZ,aAAa,yBAAyBkC;QAEtC,MAAMG,SAASC,2BAA2BJ;QAE1ClC,aAAa,sBAAsBqC,OAAO,KAAK;QAC/CrC,aAAa,yBAAyBqC,OAAO,WAAW;QAExD,IAAIE;QACJ,IAAIC,kBAAyC,EAAE;QAC/C,IAAIC,SAAmB,EAAE;QAEzB,IAAIJ,OAAO,KAAK,IAAI,CAACA,OAAO,WAAW,EAAE;YACvCI,SAAS;gBAACJ,OAAO,KAAK,IAAI;aAAoC;YAC9DrC,aAAa,yBAAyByC,MAAM,CAAC,EAAE;QACjD,OAAO;YACL,MAAM,EAAEC,CAAC,EAAEC,CAAC,EAAE,GAAGN,OAAO,WAAW;YAEnCrC,aAAa,iCAAiC;gBAAE0C;gBAAGC;YAAE;YAIrD,MAAMC,SAASC,KAAK,KAAK,CAAEH,IAAIf,aAAc;YAC7C,MAAMmB,SAASD,KAAK,KAAK,CAAEF,IAAIf,cAAe;YAE9C5B,aAAa,+BAA+B;gBAAE4C;gBAAQE;YAAO;YAG7D,IAAIC,SAASH;YACb,IAAII,SAASF;YACb,IAAIlC,QAAQ,YAAY,EAAE,MAAM;gBAC9BmC,UAAUnC,QAAQ,YAAY,CAAC,IAAI,CAAC,IAAI;gBACxCoC,UAAUpC,QAAQ,YAAY,CAAC,IAAI,CAAC,GAAG;YACzC;YAEA,MAAMqC,UAA+BC,uBACnC;gBAACH;gBAAQC;aAAO,EAChB7B;YAGFoB,UAAUU,QAAQ,IAAI;YACtBjD,aAAa,qBAAqBuC;YAElC,IAAIU,SACFT,kBAAkB;gBAACS;aAAQ;QAE/B;QAEA,OAAO;YACL,MAAMV;YACN,aAAa;gBACX,UAAUC;gBACVC;YACF;YACA,aAAaP;YACbC;YACA,mBAAmBE,OAAO,KAAK;QACjC;IACF;IAEA,IAAIc;IAKJ,IAAI;QACFA,MAAM,MAAMC,yBACV7C,MACAQ,aACA;YAAE,aAAaH,QAAQ,WAAW;QAAC;IAEvC,EAAE,OAAOyC,WAAW;QAElB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMnB,QACJkB,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAGM;QAChE,OAAO;YACL,MAAMA;YACN,aAAa;gBACX,UAAU,EAAE;gBACZ,QAAQ;oBAAC,CAAC,eAAe,EAAEL,cAAc;iBAAC;YAC5C;YACAG;YACAtB;YACA,mBAAmBwB;QACrB;IACF;IAEA,MAAMF,cAAcG,KAAK,SAAS,CAACT,IAAI,OAAO;IAE9C,IAAIZ;IACJ,IAAIC,kBAAyC,EAAE;IAC/C,IAAIC,SACF,YAAYU,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBU,MAAM,OAAO,CAACV,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;YACAZ,UAAUuB,gBACRX,IAAI,OAAO,CAAC,IAAI,EAChBxB,YACAC,aACAhB,QAAQ,YAAY,EAAE,MAAM,MAC5BA,QAAQ,YAAY,EAAE,MAAM,KAC5BiB,oBACAC,qBACAd,aACAJ,QAAQ,YAAY,EAAE;YAGxBZ,aAAa,WAAWuC;YAExB,MAAMU,UAA+Bc,sBACnCxB,SACApB;YAEFsB,SAAS,EAAE;YAEX,IAAIQ,SACFT,kBAAkB;gBAACS;aAAQ;QAE/B;IACF,EAAE,OAAOe,GAAG;QACV,MAAMC,MACJD,aAAaT,QACT,CAAC,sBAAsB,EAAES,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACvB,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEwB,IAAI,CAAC,CAAC;aAFtBxB,SAAS;YAACwB;SAAI;IAIlB;IAEA,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAgB;QACA,OAAON,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAee,gBAAgBtD,OAKrC;IAQC,MAAM,EAAEC,OAAO,EAAEsD,kBAAkB,EAAEpD,WAAW,EAAE,GAAGH;IACrD,MAAM,EAAEI,WAAW,EAAE,GAAGD;IACxB,MAAME,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAe8C,4BAA4BpD;IACjD,MAAMqD,gCAAgCC,0BACpCnE,wBAAwBgE;IAE1B,MAAM5D,OAAe;QACnB;YAAE,MAAM;YAAU,SAASe;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMoD;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMlC,SAAS,MAAM5B,mBAAmB;YACtC,QAAQ8D,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA5D,KAAK,IAAI,IAAI0B;IACf;IAEA,IAAIsC;IAGJ,IAAI;QACFA,SAAS,MAAMnB,yBACb7C,MACAQ,aACA;YAAE,aAAaH,QAAQ,WAAW;QAAC;IAEvC,EAAE,OAAOyC,WAAW;QAElB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMnB,QACJkB,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAGM;QAChE,OAAO;YACL,MAAMA;YACN,aAAaA;YACb,OAAO,CAAC,eAAe,EAAEL,cAAc;YACvCG;YACAtB;QACF;IACF;IAEA,IAAIqC;IACJ,MAAMC,cAAcF,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIE,aAAa;QACf,MAAMC,aAAaZ,gBACjBW,aACA5D,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvB,GACA,GACAA,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvBG;QAEFd,aAAa,0BAA0BwE;QAEvC,MAAMC,oBAAoBJ,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DrE,aAAa,wBAAwByE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAShB,MAAM,OAAO,CAACgB,OAC/B,GAAG,CAAC,CAACA,OACGf,gBACLe,MACAhE,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvB,GACA,GACAA,QAAQ,QAAQ,CAAC,KAAK,EACtBA,QAAQ,QAAQ,CAAC,MAAM,EACvBG;QAGNd,aAAa,qBAAqB0E;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7D1E,aAAa,iBAAiB4E;QAE9BN,cAAcQ,iBAAiBF,YAAYjE,QAAQ,QAAQ;QAC3DX,aAAa,2BAA2BsE;IAC1C;IAEA,IAAIS,cAAchE;IAClB,IAAIiE;IAEJ,IAAIV,aAAa;QACf,MAAMW,gBAAgBX,YAAY,KAAK;QACvC,MAAMY,iBAAiBZ,YAAY,MAAM;QAEzC,MAAMa,gBAAgB,MAAMC,WAC1BrE,kBACAuD,aACAxD,AAAgB,iBAAhBA;QAGF,MAAMuE,aAAa;QACnB,MAAMC,eAAe,MAAMC,WACzBJ,cAAc,WAAW,EACzBE;QAGFN,cAAcO,aAAa,WAAW;QACtCN,QAAQK;QACRf,YAAY,KAAK,GAAGgB,aAAa,KAAK;QACtChB,YAAY,MAAM,GAAGgB,aAAa,MAAM;QAExCtF,aACE,qDACAiF,eACAC,gBACAZ,YAAY,KAAK,EACjBA,YAAY,MAAM,EAClBU;IAEJ;IAEA,OAAO;QACL,MAAMV;QACNS;QACAC;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAaX,KAAK,SAAS,CAACW,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAemB,qBAAwB9E,OAO7C;IACC,MAAM,EAAE+E,SAAS,EAAE9E,OAAO,EAAE+E,aAAa,EAAEtF,gBAAgB,EAAES,WAAW,EAAE,GACxEH;IACF,MAAMU,eAAeuE;IACrB,MAAM5E,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMiF,wBAAwBC,uBAC5BnF,QAAQ,eAAe,IAAI,IAC3B+E;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAK/E;YACL,QAAQ;QACV;IACF;IAGF+E,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMvF,OAAe;QACnB;YAAE,MAAM;YAAU,SAASe;QAAa;QACxC;YACE,MAAM;YACN,SAAS0E;QACX;KACD;IAED,IAAI1F,kBAAkB;QACpB,MAAM2B,SAAS,MAAM5B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAI0B;IACf;IAEA,MAAM,EACJ,SAASwB,WAAW,EACpBtB,KAAK,EACL8D,iBAAiB,EAClB,GAAG,MAAMC,OAAO3F,MAAMQ;IAGvB,IAAIoF;IACJ,IAAI;QACFA,cAAcC,2BAA8B3C;IAC9C,EAAE,OAAO4C,YAAY;QAEnB,MAAM/C,eACJ+C,sBAAsB9C,QAAQ8C,WAAW,OAAO,GAAG7C,OAAO6C;QAC5D,MAAM,IAAI3C,qBACR,CAAC,iBAAiB,EAAEJ,cAAc,EAClCG,aACAtB;IAEJ;IAEA,OAAO;QACLgE;QACA1C;QACAtB;QACA8D;IACF;AACF;AAEO,eAAeK,sBACpBC,WAAmB,EACnBC,QAAwE,EACxEzF,WAAyB;IAKzB,MAAMO,eAAemF;IACrB,MAAMC,aAAaC,0BAA0BJ;IAE7C,MAAMhG,OAAe;QACnB;YAAE,MAAM;YAAU,SAASe;QAAa;QACxC;YACE,MAAM;YACN,SAASoF;QACX;KACD;IAED,MAAMnC,SAAS,MAAMiC,SAASjG,MAAMQ;IAEpC,OAAO;QACL,kBAAkBwD,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/llm-planning.mjs

@@ -0,0 +1,233 @@
+import { paddingToMatchBlockByBase64 } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+import { assert } from "@midscene/shared/utils";
+import { buildYamlFlowFromPlans, fillBboxParam, findAllMidsceneLocatorField } from "../common.mjs";
+import { systemPromptToTaskPlanning } from "./prompt/llm-planning.mjs";
+import { extractXMLTag, parseMarkFinishedIndexes, parseSubGoalsFromXML } from "./prompt/util.mjs";
+import { AIResponseParseError, callAI, safeParseJson } from "./service-caller/index.mjs";
+const debug = getDebug('planning');
+const warnLog = getDebug('planning', {
+    console: true
+});
+function parseXMLPlanningResponse(xmlString, modelFamily) {
+    const thought = extractXMLTag(xmlString, 'thought');
+    const memory = extractXMLTag(xmlString, 'memory');
+    const log = extractXMLTag(xmlString, 'log') || '';
+    const error = extractXMLTag(xmlString, 'error');
+    const actionType = extractXMLTag(xmlString, 'action-type');
+    const actionParamStr = extractXMLTag(xmlString, 'action-param-json');
+    const completeGoalRegex = /<complete\s+success="(true|false)">([\s\S]*?)<\/complete>/i;
+    const completeGoalMatch = xmlString.match(completeGoalRegex);
+    let finalizeMessage;
+    let finalizeSuccess;
+    if (completeGoalMatch) {
+        finalizeSuccess = 'true' === completeGoalMatch[1];
+        finalizeMessage = completeGoalMatch[2]?.trim() || void 0;
+    }
+    const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');
+    const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');
+    const updateSubGoals = updatePlanContent ? parseSubGoalsFromXML(updatePlanContent) : void 0;
+    const markFinishedIndexes = markSubGoalDone ? parseMarkFinishedIndexes(markSubGoalDone) : void 0;
+    let action = null;
+    if (actionType && 'null' !== actionType.toLowerCase()) {
+        const type = actionType.trim();
+        let param;
+        if (actionParamStr) try {
+            param = safeParseJson(actionParamStr, modelFamily);
+        } catch (e) {
+            throw new Error(`Failed to parse action-param-json: ${e}`);
+        }
+        action = {
+            type,
+            ...void 0 !== param ? {
+                param
+            } : {}
+        };
+    }
+    return {
+        ...thought ? {
+            thought
+        } : {},
+        ...memory ? {
+            memory
+        } : {},
+        log,
+        ...error ? {
+            error
+        } : {},
+        action,
+        ...void 0 !== finalizeMessage ? {
+            finalizeMessage
+        } : {},
+        ...void 0 !== finalizeSuccess ? {
+            finalizeSuccess
+        } : {},
+        ...updateSubGoals?.length ? {
+            updateSubGoals
+        } : {},
+        ...markFinishedIndexes?.length ? {
+            markFinishedIndexes
+        } : {}
+    };
+}
+async function plan(userInstruction, opts) {
+    const { context, modelConfig, conversationHistory } = opts;
+    const { shotSize } = context;
+    const screenshotBase64 = context.screenshot.base64;
+    const { modelFamily } = modelConfig;
+    const includeSubGoals = true === opts.deepThink;
+    const systemPrompt = await systemPromptToTaskPlanning({
+        actionSpace: opts.actionSpace,
+        modelFamily,
+        includeBbox: opts.includeBbox,
+        includeThought: true,
+        includeSubGoals
+    });
+    let imagePayload = screenshotBase64;
+    let imageWidth = shotSize.width;
+    let imageHeight = shotSize.height;
+    if ('qwen2.5-vl' === modelFamily) {
+        const paddedResult = await paddingToMatchBlockByBase64(imagePayload);
+        imageWidth = paddedResult.width;
+        imageHeight = paddedResult.height;
+        imagePayload = paddedResult.imageBase64;
+    }
+    const actionContext = opts.actionContext ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\n` : '';
+    const instruction = [
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`
+                }
+            ]
+        }
+    ];
+    let latestFeedbackMessage;
+    const subGoalsText = includeSubGoals ? conversationHistory.subGoalsToText() : conversationHistory.historicalLogsToText();
+    const subGoalsSection = subGoalsText ? `\n\n${subGoalsText}` : '';
+    const memoriesText = conversationHistory.memoriesToText();
+    const memoriesSection = memoriesText ? `\n\n${memoriesText}` : '';
+    if (conversationHistory.pendingFeedbackMessage) {
+        latestFeedbackMessage = {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${subGoalsSection}`
+                },
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                }
+            ]
+        };
+        conversationHistory.resetPendingFeedbackMessageIfExists();
+    } else latestFeedbackMessage = {
+        role: 'user',
+        content: [
+            {
+                type: 'text',
+                text: `this is the latest screenshot${memoriesSection}${subGoalsSection}`
+            },
+            {
+                type: 'image_url',
+                image_url: {
+                    url: imagePayload,
+                    detail: 'high'
+                }
+            }
+        ]
+    };
+    conversationHistory.append(latestFeedbackMessage);
+    conversationHistory.compressHistory(50, 20);
+    const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        ...instruction,
+        ...historyLog
+    ];
+    let { content: rawResponse, usage, reasoning_content } = await callAI(msgs, modelConfig, {
+        deepThink: 'unset' === opts.deepThink ? void 0 : opts.deepThink,
+        abortSignal: opts.abortSignal
+    });
+    let planFromAI;
+    try {
+        try {
+            planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);
+        } catch  {
+            const retry = await callAI(msgs, modelConfig, {
+                deepThink: 'unset' === opts.deepThink ? void 0 : opts.deepThink,
+                abortSignal: opts.abortSignal
+            });
+            rawResponse = retry.content;
+            usage = retry.usage;
+            reasoning_content = retry.reasoning_content;
+            planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);
+        }
+        if (planFromAI.action && void 0 !== planFromAI.finalizeSuccess) {
+            warnLog('Planning response included both an action and <complete>; ignoring <complete> output.');
+            planFromAI.finalizeMessage = void 0;
+            planFromAI.finalizeSuccess = void 0;
+        }
+        const actions = planFromAI.action ? [
+            planFromAI.action
+        ] : [];
+        let shouldContinuePlanning = true;
+        if (void 0 !== planFromAI.finalizeSuccess) {
+            debug('task completed via <complete> tag, stop planning');
+            shouldContinuePlanning = false;
+            if (includeSubGoals) conversationHistory.markAllSubGoalsFinished();
+        }
+        const returnValue = {
+            ...planFromAI,
+            actions,
+            rawResponse,
+            usage,
+            reasoning_content,
+            yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace),
+            shouldContinuePlanning
+        };
+        assert(planFromAI, "can't get plans from AI");
+        actions.forEach((action)=>{
+            const type = action.type;
+            const actionInActionSpace = opts.actionSpace.find((action)=>action.name === type);
+            debug('actionInActionSpace matched', actionInActionSpace);
+            const locateFields = actionInActionSpace ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema) : [];
+            debug('locateFields', locateFields);
+            locateFields.forEach((field)=>{
+                const locateResult = action.param[field];
+                if (locateResult && void 0 !== modelFamily) action.param[field] = fillBboxParam(locateResult, imageWidth, imageHeight, modelFamily);
+            });
+        });
+        if (includeSubGoals) {
+            if (planFromAI.updateSubGoals?.length) conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);
+            if (planFromAI.markFinishedIndexes?.length) for (const index of planFromAI.markFinishedIndexes)conversationHistory.markSubGoalFinished(index);
+            if (planFromAI.log) conversationHistory.appendSubGoalLog(planFromAI.log);
+        } else if (planFromAI.log) conversationHistory.appendHistoricalLog(planFromAI.log);
+        if (planFromAI.memory) conversationHistory.appendMemory(planFromAI.memory);
+        conversationHistory.append({
+            role: 'assistant',
+            content: [
+                {
+                    type: 'text',
+                    text: rawResponse
+                }
+            ]
+        });
+        return returnValue;
+    } catch (parseError) {
+        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
+        throw new AIResponseParseError(`XML parse error: ${errorMessage}`, rawResponse, usage);
+    }
+}
+export { parseXMLPlanningResponse, plan };
+
+//# sourceMappingURL=llm-planning.mjs.map