@aiscene/core 9.0.6 → 9.0.8

package/dist/es/ai-model/shared/xml.mjs

@@ -0,0 +1,42 @@
+function extractXMLTag(xmlString, tagName) {
+    const lowerXmlString = xmlString.toLowerCase();
+    const lowerTagName = tagName.toLowerCase();
+    const closeTag = `</${lowerTagName}>`;
+    const openTag = `<${lowerTagName}>`;
+    const lastCloseIndex = lowerXmlString.lastIndexOf(closeTag);
+    if (-1 === lastCloseIndex) {
+        const lastOpenIndex = lowerXmlString.lastIndexOf(openTag);
+        if (-1 === lastOpenIndex) return;
+        const contentStart = lastOpenIndex + openTag.length;
+        const remaining = xmlString.substring(contentStart);
+        const nextTagIndex = remaining.indexOf('<');
+        const content = -1 === nextTagIndex ? remaining : remaining.substring(0, nextTagIndex);
+        return content.trim();
+    }
+    const searchArea = lowerXmlString.substring(0, lastCloseIndex);
+    const lastOpenIndex = searchArea.lastIndexOf(openTag);
+    if (-1 === lastOpenIndex) return;
+    const contentStart = lastOpenIndex + openTag.length;
+    const contentEnd = lastCloseIndex;
+    const content = xmlString.substring(contentStart, contentEnd);
+    return content.trim();
+}
+const escapeRegExp = (value)=>value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+function extractRawXMLFragment(content, tagNames) {
+    const startTagName = tagNames[0];
+    const endTagName = tagNames.at(-1);
+    const startTagRegex = new RegExp(`<${escapeRegExp(startTagName)}>`, 'i');
+    const startTag = startTagRegex.exec(content);
+    if (startTag?.index === void 0) return '';
+    const contentFromStartTag = content.slice(startTag.index);
+    const endTagRegex = new RegExp(`</${escapeRegExp(endTagName)}>`, 'gi');
+    const endTags = [
+        ...contentFromStartTag.matchAll(endTagRegex)
+    ];
+    const lastEndTag = endTags.at(-1);
+    const end = lastEndTag?.index !== void 0 ? lastEndTag.index + lastEndTag[0].length : contentFromStartTag.length;
+    return contentFromStartTag.slice(0, end);
+}
+export { extractRawXMLFragment, extractXMLTag };
+
+//# sourceMappingURL=xml.mjs.map

package/dist/es/ai-model/shared/xml.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/xml.mjs","sources":["../../../../src/ai-model/shared/xml.ts"],"sourcesContent":["/**\n * Extract content from an XML tag in a string, searching from the end.\n * This approach handles cases where models prepend thinking content (like <think>...</think>)\n * before the actual response tags, or when there are incomplete/nested tags.\n *\n * Strategy: Find the LAST closing tag, then search backwards for the nearest opening tag.\n * This ensures we get the last complete tag pair, even if there are incomplete tags before it.\n *\n * @param xmlString - The XML string to parse\n * @param tagName - The name of the tag to extract (case-insensitive)\n * @returns The trimmed content of the tag, or undefined if not found\n */\nexport function extractXMLTag(\n  xmlString: string,\n  tagName: string,\n): string | undefined {\n  const lowerXmlString = xmlString.toLowerCase();\n  const lowerTagName = tagName.toLowerCase();\n  const closeTag = `</${lowerTagName}>`;\n  const openTag = `<${lowerTagName}>`;\n\n  // Find the last closing tag\n  const lastCloseIndex = lowerXmlString.lastIndexOf(closeTag);\n  if (lastCloseIndex === -1) {\n    // Fallback: handle half-open tags like `<action-type>Input` without\n    // matching close tag. Extract until the next XML tag boundary.\n    const lastOpenIndex = lowerXmlString.lastIndexOf(openTag);\n    if (lastOpenIndex === -1) {\n      return undefined;\n    }\n\n    const contentStart = lastOpenIndex + openTag.length;\n    const remaining = xmlString.substring(contentStart);\n    const nextTagIndex = remaining.indexOf('<');\n    const content =\n      nextTagIndex === -1 ? remaining : remaining.substring(0, nextTagIndex);\n\n    return content.trim();\n  }\n\n  // Search backwards from the closing tag to find the nearest opening tag\n  const searchArea = lowerXmlString.substring(0, lastCloseIndex);\n  const lastOpenIndex = searchArea.lastIndexOf(openTag);\n  if (lastOpenIndex === -1) {\n    return undefined;\n  }\n\n  // Extract content between the tags (use original string to preserve case)\n  const contentStart = lastOpenIndex + openTag.length;\n  const contentEnd = lastCloseIndex;\n  const content = xmlString.substring(contentStart, contentEnd);\n\n  return content.trim();\n}\n\nconst escapeRegExp = (value: string) =>\n  value.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');\n\n/** Extract raw XML from the first protocol tag through the last protocol tag. */\nexport function extractRawXMLFragment(\n  content: string,\n  tagNames: readonly [string, ...string[]],\n): string {\n  const startTagName = tagNames[0];\n  const endTagName = tagNames.at(-1)!;\n  const startTagRegex = new RegExp(`<${escapeRegExp(startTagName)}>`, 'i');\n  const startTag = startTagRegex.exec(content);\n  if (startTag?.index === undefined) {\n    return '';\n  }\n\n  const contentFromStartTag = content.slice(startTag.index);\n  const endTagRegex = new RegExp(`</${escapeRegExp(endTagName)}>`, 'gi');\n  const endTags = [...contentFromStartTag.matchAll(endTagRegex)];\n  const lastEndTag = endTags.at(-1);\n  const end =\n    lastEndTag?.index !== undefined\n      ? lastEndTag.index + lastEndTag[0].length\n      : contentFromStartTag.length;\n\n  return contentFromStartTag.slice(0, end);\n}\n"],"names":["extractXMLTag","xmlString","tagName","lowerXmlString","lowerTagName","closeTag","openTag","lastCloseIndex","lastOpenIndex","contentStart","remaining","nextTagIndex","content","searchArea","contentEnd","escapeRegExp","value","extractRawXMLFragment","tagNames","startTagName","endTagName","startTagRegex","RegExp","startTag","undefined","contentFromStartTag","endTagRegex","endTags","lastEndTag","end"],"mappings":"AAYO,SAASA,cACdC,SAAiB,EACjBC,OAAe;IAEf,MAAMC,iBAAiBF,UAAU,WAAW;IAC5C,MAAMG,eAAeF,QAAQ,WAAW;IACxC,MAAMG,WAAW,CAAC,EAAE,EAAED,aAAa,CAAC,CAAC;IACrC,MAAME,UAAU,CAAC,CAAC,EAAEF,aAAa,CAAC,CAAC;IAGnC,MAAMG,iBAAiBJ,eAAe,WAAW,CAACE;IAClD,IAAIE,AAAmB,OAAnBA,gBAAuB;QAGzB,MAAMC,gBAAgBL,eAAe,WAAW,CAACG;QACjD,IAAIE,AAAkB,OAAlBA,eACF;QAGF,MAAMC,eAAeD,gBAAgBF,QAAQ,MAAM;QACnD,MAAMI,YAAYT,UAAU,SAAS,CAACQ;QACtC,MAAME,eAAeD,UAAU,OAAO,CAAC;QACvC,MAAME,UACJD,AAAiB,OAAjBA,eAAsBD,YAAYA,UAAU,SAAS,CAAC,GAAGC;QAE3D,OAAOC,QAAQ,IAAI;IACrB;IAGA,MAAMC,aAAaV,eAAe,SAAS,CAAC,GAAGI;IAC/C,MAAMC,gBAAgBK,WAAW,WAAW,CAACP;IAC7C,IAAIE,AAAkB,OAAlBA,eACF;IAIF,MAAMC,eAAeD,gBAAgBF,QAAQ,MAAM;IACnD,MAAMQ,aAAaP;IACnB,MAAMK,UAAUX,UAAU,SAAS,CAACQ,cAAcK;IAElD,OAAOF,QAAQ,IAAI;AACrB;AAEA,MAAMG,eAAe,CAACC,QACpBA,MAAM,OAAO,CAAC,uBAAuB;AAGhC,SAASC,sBACdL,OAAe,EACfM,QAAwC;IAExC,MAAMC,eAAeD,QAAQ,CAAC,EAAE;IAChC,MAAME,aAAaF,SAAS,EAAE,CAAC;IAC/B,MAAMG,gBAAgB,IAAIC,OAAO,CAAC,CAAC,EAAEP,aAAaI,cAAc,CAAC,CAAC,EAAE;IACpE,MAAMI,WAAWF,cAAc,IAAI,CAACT;IACpC,IAAIW,UAAU,UAAUC,QACtB,OAAO;IAGT,MAAMC,sBAAsBb,QAAQ,KAAK,CAACW,SAAS,KAAK;IACxD,MAAMG,cAAc,IAAIJ,OAAO,CAAC,EAAE,EAAEP,aAAaK,YAAY,CAAC,CAAC,EAAE;IACjE,MAAMO,UAAU;WAAIF,oBAAoB,QAAQ,CAACC;KAAa;IAC9D,MAAME,aAAaD,QAAQ,EAAE,CAAC;IAC9B,MAAME,MACJD,YAAY,UAAUJ,SAClBI,WAAW,KAAK,GAAGA,UAAU,CAAC,EAAE,CAAC,MAAM,GACvCH,oBAAoB,MAAM;IAEhC,OAAOA,oBAAoB,KAAK,CAAC,GAAGI;AACtC"}

package/dist/es/ai-model/workflows/grounding/index.mjs

@@ -0,0 +1,4 @@
+import { AiLocateElement } from "./locate.mjs";
+import { AiLocateSection } from "./section-locate.mjs";
+import { buildSearchAreaConfig } from "./search-area.mjs";
+export { AiLocateElement, AiLocateSection, buildSearchAreaConfig };

package/dist/es/ai-model/workflows/grounding/locate-result-rect.mjs

@@ -0,0 +1,23 @@
+function mergePixelBboxesToRect(pixelBboxes) {
+    const minLeft = Math.min(...pixelBboxes.map(([left])=>left));
+    const minTop = Math.min(...pixelBboxes.map(([, top])=>top));
+    const maxRight = Math.max(...pixelBboxes.map(([, , right])=>right));
+    const maxBottom = Math.max(...pixelBboxes.map(([, , , bottom])=>bottom));
+    return pixelBboxToRect([
+        minLeft,
+        minTop,
+        maxRight,
+        maxBottom
+    ]);
+}
+function pixelBboxToRect([left, top, right, bottom]) {
+    return {
+        left,
+        top,
+        width: right - left + 1,
+        height: bottom - top + 1
+    };
+}
+export { mergePixelBboxesToRect, pixelBboxToRect };
+
+//# sourceMappingURL=locate-result-rect.mjs.map

package/dist/es/ai-model/workflows/grounding/locate-result-rect.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/locate-result-rect.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate-result-rect.ts"],"sourcesContent":["import type { PixelBbox } from '@/ai-model/shared/model-locate-result';\nimport type { Rect } from '@/types';\n\n/** Merge model pixel bboxes into one screenshot rectangle. */\nexport function mergePixelBboxesToRect(pixelBboxes: PixelBbox[]): Rect {\n  const minLeft = Math.min(...pixelBboxes.map(([left]) => left));\n  const minTop = Math.min(...pixelBboxes.map(([, top]) => top));\n  const maxRight = Math.max(...pixelBboxes.map(([, , right]) => right));\n  const maxBottom = Math.max(...pixelBboxes.map(([, , , bottom]) => bottom));\n  return pixelBboxToRect([minLeft, minTop, maxRight, maxBottom]);\n}\n\nexport function pixelBboxToRect([left, top, right, bottom]: PixelBbox): Rect {\n  return {\n    left,\n    top,\n    width: right - left + 1,\n    height: bottom - top + 1,\n  };\n}\n"],"names":["mergePixelBboxesToRect","pixelBboxes","minLeft","Math","left","minTop","top","maxRight","right","maxBottom","bottom","pixelBboxToRect"],"mappings":"AAIO,SAASA,uBAAuBC,WAAwB;IAC7D,MAAMC,UAAUC,KAAK,GAAG,IAAIF,YAAY,GAAG,CAAC,CAAC,CAACG,KAAK,GAAKA;IACxD,MAAMC,SAASF,KAAK,GAAG,IAAIF,YAAY,GAAG,CAAC,CAAC,GAAGK,IAAI,GAAKA;IACxD,MAAMC,WAAWJ,KAAK,GAAG,IAAIF,YAAY,GAAG,CAAC,CAAC,KAAKO,MAAM,GAAKA;IAC9D,MAAMC,YAAYN,KAAK,GAAG,IAAIF,YAAY,GAAG,CAAC,CAAC,OAAOS,OAAO,GAAKA;IAClE,OAAOC,gBAAgB;QAACT;QAASG;QAAQE;QAAUE;KAAU;AAC/D;AAEO,SAASE,gBAAgB,CAACP,MAAME,KAAKE,OAAOE,OAAkB;IACnE,OAAO;QACLN;QACAE;QACA,OAAOE,QAAQJ,OAAO;QACtB,QAAQM,SAASJ,MAAM;IACzB;AACF"}

package/dist/es/ai-model/workflows/grounding/locate.mjs

@@ -0,0 +1,168 @@
+import { generateElementByRect } from "@aiscene/shared/extractor";
+import { getDebug } from "@aiscene/shared/logger";
+import { assert } from "@aiscene/shared/utils";
+import { userPromptToString } from "../../../common.mjs";
+import { buildElementLocateSystemPrompt } from "../../prompt/locate/index.mjs";
+import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
+import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
+import { pixelBboxToRect } from "./locate-result-rect.mjs";
+import { mapSearchAreaPixelBboxToOriginalPixelBbox } from "./search-area-mapping.mjs";
+import { formatLocateModelContext, prepareLocateModelInput } from "./utils.mjs";
+const debugGrounding = getDebug('ai:grounding');
+async function AiLocateElement(options) {
+    const { targetElementDescription, ...locateOptions } = options;
+    assert(targetElementDescription, "cannot find the target element description");
+    const { context } = locateOptions;
+    const locateImage = locateOptions.searchConfig?.image ?? {
+        imageBase64: context.screenshot.base64,
+        width: context.shotSize.width,
+        height: context.shotSize.height
+    };
+    const locateRequest = {
+        targetElementDescription,
+        locateImage,
+        options: locateOptions
+    };
+    const locateAdapter = options.modelRuntime.adapter.locate;
+    const locateFn = 'custom' === locateAdapter.kind ? locateAdapter.locateFn : genericLocate;
+    const locateResponse = await locateFn(locateRequest);
+    const { locatedPixelBbox, rawResponse, rawChoiceMessage, usage, reasoningContent, errors = [] } = locateResponse;
+    const baseLocateResult = {
+        rawResponse,
+        rawChoiceMessage,
+        usage,
+        reasoning_content: reasoningContent
+    };
+    if (!locatedPixelBbox) return {
+        rect: void 0,
+        parseResult: {
+            element: void 0,
+            errors
+        },
+        ...baseLocateResult
+    };
+    try {
+        const rect = pixelBboxToRect(mapSearchAreaPixelBboxToOriginalPixelBbox(locatedPixelBbox, locateOptions.searchConfig?.mapping));
+        debugGrounding('resRect', rect);
+        return {
+            rect,
+            parseResult: {
+                element: generateElementByRect(rect, userPromptToString(targetElementDescription)),
+                errors: []
+            },
+            ...baseLocateResult
+        };
+    } catch (error) {
+        const msg = error instanceof Error ? `Failed to parse locate result: ${error.message}` : 'unknown error in locate';
+        return {
+            rect: void 0,
+            parseResult: {
+                element: void 0,
+                errors: errors.length > 0 ? [
+                    ...errors,
+                    `(${msg})`
+                ] : [
+                    msg
+                ]
+            },
+            ...baseLocateResult
+        };
+    }
+}
+async function genericLocate(locateRequest) {
+    const { options, targetElementDescription } = locateRequest;
+    const modelRuntime = options.modelRuntime;
+    const { adapter } = modelRuntime;
+    assert('standard' === adapter.locate.kind, 'generic locate requires a standard locate adapter');
+    const { protocol, resultCodec } = adapter.locate.element;
+    const systemPrompt = buildElementLocateSystemPrompt({
+        systemPromptIntroduction: protocol.systemPromptIntroduction,
+        responseInstructions: protocol.buildResponseInstructions(resultCodec.promptSpec)
+    });
+    const userInstructionPrompt = protocol.buildUserPrompt(userPromptToString(targetElementDescription));
+    const { messages, preparedImage } = await prepareLocateModelInput({
+        systemPrompt,
+        userPrompt: userInstructionPrompt,
+        locateImage: locateRequest.locateImage,
+        imagePreprocess: adapter.imagePreprocess,
+        targetDescription: targetElementDescription,
+        userMessageContentOrder: adapter.locate.userMessageContentOrder
+    });
+    try {
+        return await callAiAndParseWithRetry({
+            callAi: (retryAttempt, previousParseError)=>callAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
+                    abortSignal: options.abortSignal,
+                    expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,
+                    semanticRetryAttempt: retryAttempt
+                }),
+            parseResponse: (response)=>{
+                const parsedLocateResult = protocol.parseRawResponse(response.content, resultCodec.promptSpec);
+                const rawResponse = response.content;
+                const locateError = parsedLocateResult.error;
+                if ('not-found' === parsedLocateResult.kind) return {
+                    rawResponse,
+                    rawChoiceMessage: response.rawChoiceMessage,
+                    usage: response.usage,
+                    reasoningContent: response.reasoning_content,
+                    errors: locateError ? [
+                        locateError
+                    ] : []
+                };
+                try {
+                    const locatedPixelBbox = resultCodec.toPixelBbox(parsedLocateResult.target, {
+                        preparedSize: preparedImage.preparedSize,
+                        contentSize: preparedImage.contentSize
+                    });
+                    return {
+                        locatedPixelBbox,
+                        rawResponse,
+                        rawChoiceMessage: response.rawChoiceMessage,
+                        usage: response.usage,
+                        reasoningContent: response.reasoning_content,
+                        errors: []
+                    };
+                } catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    const message = [
+                        locateError && `error in locate result: ${locateError}`,
+                        `coordinate parsing error: ${errorMessage}`,
+                        formatLocateModelContext(modelRuntime)
+                    ].filter(Boolean).join('\n');
+                    throw new Error(message, {
+                        cause: error
+                    });
+                }
+            },
+            toParseError: (error, response)=>{
+                const parseErrorMessage = error instanceof Error ? `Failed to parse locate result: ${error.message}` : 'unknown error in locate result';
+                return new AIResponseParseError(parseErrorMessage, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
+            },
+            parseRetryTimes: modelRuntime.config.retryCount,
+            parseRetryInterval: modelRuntime.config.retryInterval,
+            abortSignal: options.abortSignal,
+            onParseRetry: (error)=>{
+                debugGrounding('retrying locate after coordinate parsing failed: %s', error instanceof Error ? error.message : String(error));
+            }
+        });
+    } catch (callError) {
+        if (callError instanceof AIResponseParseError) return {
+            rawResponse: callError.rawResponse,
+            rawChoiceMessage: callError.rawChoiceMessage,
+            usage: callError.usage,
+            reasoningContent: callError.reasoningContent,
+            errors: [
+                callError.message
+            ]
+        };
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        return {
+            rawResponse: errorMessage,
+            errors: [
+                `AI call error: ${errorMessage}`
+            ]
+        };
+    }
+}
+export { AiLocateElement, userPromptToString as extraTextFromUserPrompt, genericLocate };
+
+//# sourceMappingURL=locate.mjs.map

package/dist/es/ai-model/workflows/grounding/locate.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { generateElementByRect } from '@aiscene/shared/extractor';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert } from '@aiscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { pixelBboxToRect } from './locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugGrounding('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          userPromptToString(targetElementDescription),\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const locatedPixelBbox = resultCodec.toPixelBbox(\n            parsedLocateResult.target,\n            {\n              preparedSize: preparedImage.preparedSize,\n              contentSize: preparedImage.contentSize,\n            },\n          );\n          return {\n            locatedPixelBbox,\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","undefined","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;;;AAqBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAMO;QACN,aAAa;YACX,SAASA;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,OAAOC,gBACXC,0CACEV,kBACAT,cAAc,YAAY,EAAE;QAGhCL,eAAe,WAAWsB;QAE1B,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,sBACPH,MACAI,mBAAmBtB;gBAErB,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOO,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMN;YACN,aAAa;gBACX,SAASA;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAES,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGR,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMqB,eAAe3B,QAAQ,YAAY;IACzC,MAAM,EAAE4B,OAAO,EAAE,GAAGD;IACpBxB,OACEyB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBtB;IAGrB,MAAM,EAAEiC,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAa3B,cAAc,WAAW;QACtC,iBAAiBsB,QAAQ,eAAe;QACxC,mBAAmB3B;QACnB,yBAAyB2B,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAa3B,QAAQ,WAAW;oBAChC,4BAA4B6B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMlB,cAAc8B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL/B;oBACA,kBAAkB8B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMjC,mBAAmBmB,YAAY,WAAW,CAC9Ca,mBAAmB,MAAM,EACzB;wBACE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBAEF,OAAO;wBACLxB;wBACAC;wBACA,kBAAkB8B,SAAS,gBAAgB;wBAC3C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMqB,eACJrB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;oBAClD,MAAMuB,UAAU;wBACdH,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEC,cAAc;wBAC3CG,yBAAyBrB;qBAC1B,CACE,MAAM,CAACsB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIvB,MAAMqB,SAAS;wBAAE,OAAOvB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMQ,oBACJ1B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI2B,qBACTD,mBACAR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAa3B,QAAQ,WAAW;YAChC,cAAc,CAACwB;gBACb3B,eACE,uDACA2B,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAO4B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB1B,QAAQ0B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/planning-action-locate.mjs

@@ -0,0 +1,97 @@
+import { getDebug } from "@aiscene/shared/logger";
+import { assert } from "@aiscene/shared/utils";
+import { z } from "zod";
+import { getMidsceneLocationSchema } from "../../../common.mjs";
+import { ScreenshotItem } from "../../../screenshot-item.mjs";
+import { AIResponseParseError } from "../../service-caller/index.mjs";
+import { prepareUserPrompt } from "../../shared/multimodal-prompt.mjs";
+import { ConversationHistory } from "../planning/conversation-history.mjs";
+import { runCustomPlanning } from "../planning/custom-planning.mjs";
+const debugGrounding = getDebug('ai:grounding');
+const planningActionLocatorActionSpace = [
+    {
+        name: 'Tap',
+        description: 'Tap the element',
+        paramSchema: z.object({
+            locate: getMidsceneLocationSchema()
+        }),
+        call: async ()=>void 0
+    }
+];
+async function buildPlanningTapLocatorPlanOptions(locateRequest) {
+    const { options, locateImage } = locateRequest;
+    const { context } = options;
+    return {
+        ...options,
+        context: {
+            ...context,
+            screenshot: ScreenshotItem.create(locateImage.imageBase64, context.screenshot.capturedAt),
+            shotSize: {
+                width: locateImage.width,
+                height: locateImage.height
+            }
+        },
+        actionSpace: planningActionLocatorActionSpace,
+        conversationHistory: new ConversationHistory(),
+        includeLocateInPlanning: true,
+        effort: 'balance'
+    };
+}
+function resolvePlanningTapLocator(definition, planner) {
+    const locatorPlanner = {
+        ...planner,
+        messages: {
+            ...planner.messages,
+            buildSystemPrompt: definition.buildSystemPrompt,
+            buildUserInstruction: (instruction)=>`Tap: ${instruction}`
+        }
+    };
+    return async (locateRequest)=>{
+        const { targetElementDescription } = locateRequest;
+        assert(targetElementDescription, "cannot find the target element description");
+        let errors = [];
+        let reasoningContent = '';
+        let rawResponse = '';
+        let rawChoiceMessage;
+        let usage;
+        try {
+            const locatePlanOptions = await buildPlanningTapLocatorPlanOptions(locateRequest);
+            const planningResponse = await runCustomPlanning(await prepareUserPrompt(targetElementDescription), locatePlanOptions, locatorPlanner);
+            rawResponse = planningResponse.rawResponse ?? '';
+            rawChoiceMessage = planningResponse.rawChoiceMessage;
+            usage = planningResponse.usage;
+            reasoningContent = planningResponse.log;
+            debugGrounding('planning-tap-locator rawResponse:', rawResponse);
+            const locatedPixelBbox = definition.getLocatedPixelBbox(planningResponse.actions ?? []);
+            if (!locatedPixelBbox) throw new Error('No locatedPixelBbox found in planner response');
+            return {
+                locatedPixelBbox,
+                rawResponse,
+                rawChoiceMessage,
+                usage,
+                reasoningContent
+            };
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            if (error instanceof AIResponseParseError) {
+                rawResponse = error.rawResponse;
+                rawChoiceMessage = error.rawChoiceMessage;
+                usage = error.usage;
+            }
+            errors = [
+                errorMessage || 'Failed to parse planning tap locator response'
+            ];
+            debugGrounding('planning-tap-locator parse error:', errors[0]);
+        }
+        return {
+            rawResponse,
+            rawChoiceMessage,
+            usage,
+            reasoningContent,
+            errors
+        };
+    };
+}
+export { resolvePlanningTapLocator };
+
+//# sourceMappingURL=planning-action-locate.mjs.map

package/dist/es/ai-model/workflows/grounding/planning-action-locate.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/planning-action-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/planning-action-locate.ts"],"sourcesContent":["import type { DeviceAction } from '@/device';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert } from '@aiscene/shared/utils';\nimport { z } from 'zod';\nimport { getMidsceneLocationSchema } from '../../../common';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type { ResolvedCustomPlanningDefinition } from '../../model-adapter/custom-planning-types';\nimport type { PlanningTapLocatorDefinition } from '../../model-adapter/types';\nimport { AIResponseParseError } from '../../service-caller/index';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\nimport { ConversationHistory } from '../planning/conversation-history';\nimport { runCustomPlanning } from '../planning/custom-planning';\nimport type { PlanOptions } from '../planning/types';\nimport type { LocateFn, LocateModelResponse, LocateRequest } from './types';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nconst planningActionLocatorActionSpace: DeviceAction[] = [\n  {\n    name: 'Tap',\n    description: 'Tap the element',\n    paramSchema: z.object({\n      locate: getMidsceneLocationSchema(),\n    }),\n    call: async () => undefined,\n  },\n];\n\nasync function buildPlanningTapLocatorPlanOptions(\n  locateRequest: LocateRequest,\n): Promise<PlanOptions> {\n  const { options, locateImage } = locateRequest;\n  const { context } = options;\n\n  return {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        locateImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: {\n        width: locateImage.width,\n        height: locateImage.height,\n      },\n    },\n    actionSpace: planningActionLocatorActionSpace,\n    conversationHistory: new ConversationHistory(),\n    includeLocateInPlanning: true,\n    effort: 'balance',\n  };\n}\n\nexport function resolvePlanningTapLocator<TParsed>(\n  definition: PlanningTapLocatorDefinition,\n  planner: ResolvedCustomPlanningDefinition<TParsed>,\n): LocateFn {\n  const locatorPlanner: ResolvedCustomPlanningDefinition<TParsed> = {\n    ...planner,\n    messages: {\n      ...planner.messages,\n      buildSystemPrompt: definition.buildSystemPrompt,\n      buildUserInstruction: (instruction) => `Tap: ${instruction}`,\n    },\n  };\n\n  return async (locateRequest: LocateRequest): Promise<LocateModelResponse> => {\n    const { targetElementDescription } = locateRequest;\n    assert(\n      targetElementDescription,\n      'cannot find the target element description',\n    );\n\n    let errors: string[] = [];\n    let reasoningContent = '';\n    let rawResponse = '';\n    let rawChoiceMessage: unknown;\n    let usage: LocateModelResponse['usage'];\n\n    try {\n      const locatePlanOptions =\n        await buildPlanningTapLocatorPlanOptions(locateRequest);\n      const planningResponse = await runCustomPlanning(\n        await prepareUserPrompt(targetElementDescription),\n        locatePlanOptions,\n        locatorPlanner,\n      );\n\n      rawResponse = planningResponse.rawResponse ?? '';\n      rawChoiceMessage = planningResponse.rawChoiceMessage;\n      usage = planningResponse.usage;\n      reasoningContent = planningResponse.log;\n\n      debugGrounding('planning-tap-locator rawResponse:', rawResponse);\n\n      const locatedPixelBbox = definition.getLocatedPixelBbox(\n        planningResponse.actions ?? [],\n      );\n\n      if (!locatedPixelBbox) {\n        throw new Error('No locatedPixelBbox found in planner response');\n      }\n\n      return {\n        locatedPixelBbox,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoningContent,\n      };\n    } catch (error) {\n      const errorMessage =\n        error instanceof Error ? error.message : String(error);\n      if (error instanceof AIResponseParseError) {\n        rawResponse = error.rawResponse;\n        rawChoiceMessage = error.rawChoiceMessage;\n        usage = error.usage;\n      }\n      errors = [\n        errorMessage || 'Failed to parse planning tap locator response',\n      ];\n      debugGrounding('planning-tap-locator parse error:', errors[0]);\n    }\n\n    return {\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoningContent,\n      errors,\n    };\n  };\n}\n"],"names":["debugGrounding","getDebug","planningActionLocatorActionSpace","z","getMidsceneLocationSchema","undefined","buildPlanningTapLocatorPlanOptions","locateRequest","options","locateImage","context","ScreenshotItem","ConversationHistory","resolvePlanningTapLocator","definition","planner","locatorPlanner","instruction","targetElementDescription","assert","errors","reasoningContent","rawResponse","rawChoiceMessage","usage","locatePlanOptions","planningResponse","runCustomPlanning","prepareUserPrompt","locatedPixelBbox","Error","error","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;;;AAeA,MAAMA,iBAAiBC,SAAS;AAEhC,MAAMC,mCAAmD;IACvD;QACE,MAAM;QACN,aAAa;QACb,aAAaC,EAAE,MAAM,CAAC;YACpB,QAAQC;QACV;QACA,MAAM,UAAYC;IACpB;CACD;AAED,eAAeC,mCACbC,aAA4B;IAE5B,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAE,GAAGF;IACjC,MAAM,EAAEG,OAAO,EAAE,GAAGF;IAEpB,OAAO;QACL,GAAGA,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYC,eAAe,MAAM,CAC/BF,YAAY,WAAW,EACvBC,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAU;gBACR,OAAOD,YAAY,KAAK;gBACxB,QAAQA,YAAY,MAAM;YAC5B;QACF;QACA,aAAaP;QACb,qBAAqB,IAAIU;QACzB,yBAAyB;QACzB,QAAQ;IACV;AACF;AAEO,SAASC,0BACdC,UAAwC,EACxCC,OAAkD;IAElD,MAAMC,iBAA4D;QAChE,GAAGD,OAAO;QACV,UAAU;YACR,GAAGA,QAAQ,QAAQ;YACnB,mBAAmBD,WAAW,iBAAiB;YAC/C,sBAAsB,CAACG,cAAgB,CAAC,KAAK,EAAEA,aAAa;QAC9D;IACF;IAEA,OAAO,OAAOV;QACZ,MAAM,EAAEW,wBAAwB,EAAE,GAAGX;QACrCY,OACED,0BACA;QAGF,IAAIE,SAAmB,EAAE;QACzB,IAAIC,mBAAmB;QACvB,IAAIC,cAAc;QAClB,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMC,oBACJ,MAAMnB,mCAAmCC;YAC3C,MAAMmB,mBAAmB,MAAMC,kBAC7B,MAAMC,kBAAkBV,2BACxBO,mBACAT;YAGFM,cAAcI,iBAAiB,WAAW,IAAI;YAC9CH,mBAAmBG,iBAAiB,gBAAgB;YACpDF,QAAQE,iBAAiB,KAAK;YAC9BL,mBAAmBK,iBAAiB,GAAG;YAEvC1B,eAAe,qCAAqCsB;YAEpD,MAAMO,mBAAmBf,WAAW,mBAAmB,CACrDY,iBAAiB,OAAO,IAAI,EAAE;YAGhC,IAAI,CAACG,kBACH,MAAM,IAAIC,MAAM;YAGlB,OAAO;gBACLD;gBACAP;gBACAC;gBACAC;gBACAH;YACF;QACF,EAAE,OAAOU,OAAO;YACd,MAAMC,eACJD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;YAClD,IAAIA,iBAAiBG,sBAAsB;gBACzCZ,cAAcS,MAAM,WAAW;gBAC/BR,mBAAmBQ,MAAM,gBAAgB;gBACzCP,QAAQO,MAAM,KAAK;YACrB;YACAX,SAAS;gBACPY,gBAAgB;aACjB;YACDhC,eAAe,qCAAqCoB,MAAM,CAAC,EAAE;QAC/D;QAEA,OAAO;YACLE;YACAC;YACAC;YACAH;YACAD;QACF;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/search-area-mapping.mjs

@@ -0,0 +1,18 @@
+function mapSearchAreaPixelBboxToOriginalPixelBbox([left, top, right, bottom], mapping) {
+    const offset = mapping?.offset ?? {
+        x: 0,
+        y: 0
+    };
+    const scale = mapping?.scale ?? 1;
+    const mapX = (x)=>(1 !== scale ? Math.round(x / scale) : x) + offset.x;
+    const mapY = (y)=>(1 !== scale ? Math.round(y / scale) : y) + offset.y;
+    return [
+        mapX(left),
+        mapY(top),
+        mapX(right),
+        mapY(bottom)
+    ];
+}
+export { mapSearchAreaPixelBboxToOriginalPixelBbox };
+
+//# sourceMappingURL=search-area-mapping.mjs.map

package/dist/es/ai-model/workflows/grounding/search-area-mapping.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/search-area-mapping.mjs","sources":["../../../../../src/ai-model/workflows/grounding/search-area-mapping.ts"],"sourcesContent":["import type { PixelBbox } from '@/ai-model/shared/model-locate-result';\nimport type { SearchAreaImageMapping } from './types';\n\n/** Map a bbox in the prepared search-area image back to screenshot pixels. */\nexport function mapSearchAreaPixelBboxToOriginalPixelBbox(\n  [left, top, right, bottom]: PixelBbox,\n  mapping?: SearchAreaImageMapping,\n): PixelBbox {\n  const offset = mapping?.offset ?? { x: 0, y: 0 };\n  const scale = mapping?.scale ?? 1;\n  const mapX = (x: number) =>\n    (scale !== 1 ? Math.round(x / scale) : x) + offset.x;\n  const mapY = (y: number) =>\n    (scale !== 1 ? Math.round(y / scale) : y) + offset.y;\n\n  return [mapX(left), mapY(top), mapX(right), mapY(bottom)];\n}\n"],"names":["mapSearchAreaPixelBboxToOriginalPixelBbox","left","top","right","bottom","mapping","offset","scale","mapX","x","Math","mapY","y"],"mappings":"AAIO,SAASA,0CACd,CAACC,MAAMC,KAAKC,OAAOC,OAAkB,EACrCC,OAAgC;IAEhC,MAAMC,SAASD,SAAS,UAAU;QAAE,GAAG;QAAG,GAAG;IAAE;IAC/C,MAAME,QAAQF,SAAS,SAAS;IAChC,MAAMG,OAAO,CAACC,IACXF,AAAAA,CAAAA,AAAU,MAAVA,QAAcG,KAAK,KAAK,CAACD,IAAIF,SAASE,CAAAA,IAAKH,OAAO,CAAC;IACtD,MAAMK,OAAO,CAACC,IACXL,AAAAA,CAAAA,AAAU,MAAVA,QAAcG,KAAK,KAAK,CAACE,IAAIL,SAASK,CAAAA,IAAKN,OAAO,CAAC;IAEtD,OAAO;QAACE,KAAKP;QAAOU,KAAKT;QAAMM,KAAKL;QAAQQ,KAAKP;KAAQ;AAC3D"}

package/dist/es/ai-model/workflows/grounding/search-area.mjs

@@ -0,0 +1,55 @@
+import { cropByRect, scaleImage } from "@aiscene/shared/img";
+function expandSearchArea(rect, screenSize) {
+    const minArea = 160000;
+    const expandSize = 100;
+    const expandedLeft = Math.max(rect.left - expandSize, 0);
+    const expandedTop = Math.max(rect.top - expandSize, 0);
+    const expandRect = {
+        left: expandedLeft,
+        top: expandedTop,
+        width: Math.min(rect.left - expandedLeft + rect.width + expandSize, screenSize.width - expandedLeft),
+        height: Math.min(rect.top - expandedTop + rect.height + expandSize, screenSize.height - expandedTop)
+    };
+    const currentArea = expandRect.width * expandRect.height;
+    if (currentArea >= minArea) return expandRect;
+    const centerX = expandRect.left + expandRect.width / 2;
+    const centerY = expandRect.top + expandRect.height / 2;
+    const scaleFactor = Math.sqrt(minArea / currentArea);
+    const newWidth = Math.round(expandRect.width * scaleFactor);
+    const newHeight = Math.round(expandRect.height * scaleFactor);
+    const newLeft = Math.round(centerX - newWidth / 2);
+    const newTop = Math.round(centerY - newHeight / 2);
+    const left = Math.max(newLeft, 0);
+    const top = Math.max(newTop, 0);
+    return {
+        left,
+        top,
+        width: Math.min(newWidth, screenSize.width - left),
+        height: Math.min(newHeight, screenSize.height - top)
+    };
+}
+async function buildSearchAreaConfig(options) {
+    const { context, baseRect } = options;
+    const scaleRatio = 2;
+    const sectionRect = expandSearchArea(baseRect, context.shotSize);
+    const croppedResult = await cropByRect(context.screenshot.base64, sectionRect);
+    const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);
+    return {
+        sourceRect: sectionRect,
+        image: {
+            imageBase64: scaledResult.imageBase64,
+            width: scaledResult.width,
+            height: scaledResult.height
+        },
+        mapping: {
+            offset: {
+                x: sectionRect.left,
+                y: sectionRect.top
+            },
+            scale: scaleRatio
+        }
+    };
+}
+export { buildSearchAreaConfig, expandSearchArea };
+
+//# sourceMappingURL=search-area.mjs.map

package/dist/es/ai-model/workflows/grounding/search-area.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/search-area.mjs","sources":["../../../../../src/ai-model/workflows/grounding/search-area.ts"],"sourcesContent":["import type { Rect, Size, UIContext } from '@/types';\nimport { cropByRect, scaleImage } from '@aiscene/shared/img';\nimport type { SearchAreaConfig } from './types';\n\n/**\n * Expand the search area to at least 400 x 400 pixels\n *\n * Step 1: Extend 100px on each side (top, right, bottom, left)\n * - If the element is near a boundary, expansion on that side will be limited\n * - No compensation is made for boundary limitations (this is intentional)\n *\n * Step 2: Ensure the area is at least 400x400 pixels\n * - Scale up proportionally from the center if needed\n * - Final result is clamped to screen boundaries\n */\nexport function expandSearchArea(rect: Rect, screenSize: Size): Rect {\n  const minArea = 400 * 400;\n  const expandSize = 100;\n\n  // Step 1: Extend each side by expandSize (100px), clamped to screen boundaries\n  // Note: If element is near boundary, actual expansion may be less than 100px on that side\n  const expandedLeft = Math.max(rect.left - expandSize, 0);\n  const expandedTop = Math.max(rect.top - expandSize, 0);\n\n  const expandRect = {\n    left: expandedLeft,\n    top: expandedTop,\n    width: Math.min(\n      rect.left - expandedLeft + rect.width + expandSize,\n      screenSize.width - expandedLeft,\n    ),\n    height: Math.min(\n      rect.top - expandedTop + rect.height + expandSize,\n      screenSize.height - expandedTop,\n    ),\n  };\n\n  // Step 2: Check if area is already >= 400x400\n  const currentArea = expandRect.width * expandRect.height;\n\n  if (currentArea >= minArea) {\n    return expandRect;\n  }\n\n  // Step 2: Scale up from center to reach minimum 400x400 area\n  const centerX = expandRect.left + expandRect.width / 2;\n  const centerY = expandRect.top + expandRect.height / 2;\n\n  // Calculate scale factor needed to reach minimum area\n  const scaleFactor = Math.sqrt(minArea / currentArea);\n  const newWidth = Math.round(expandRect.width * scaleFactor);\n  const newHeight = Math.round(expandRect.height * scaleFactor);\n\n  // Calculate new position based on center point\n  const newLeft = Math.round(centerX - newWidth / 2);\n  const newTop = Math.round(centerY - newHeight / 2);\n\n  // Clamp to screen boundaries\n  const left = Math.max(newLeft, 0);\n  const top = Math.max(newTop, 0);\n\n  return {\n    left,\n    top,\n    width: Math.min(newWidth, screenSize.width - left),\n    height: Math.min(newHeight, screenSize.height - top),\n  };\n}\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n"],"names":["expandSearchArea","rect","screenSize","minArea","expandSize","expandedLeft","Math","expandedTop","expandRect","currentArea","centerX","centerY","scaleFactor","newWidth","newHeight","newLeft","newTop","left","top","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","croppedResult","cropByRect","scaledResult","scaleImage"],"mappings":";AAeO,SAASA,iBAAiBC,IAAU,EAAEC,UAAgB;IAC3D,MAAMC,UAAU;IAChB,MAAMC,aAAa;IAInB,MAAMC,eAAeC,KAAK,GAAG,CAACL,KAAK,IAAI,GAAGG,YAAY;IACtD,MAAMG,cAAcD,KAAK,GAAG,CAACL,KAAK,GAAG,GAAGG,YAAY;IAEpD,MAAMI,aAAa;QACjB,MAAMH;QACN,KAAKE;QACL,OAAOD,KAAK,GAAG,CACbL,KAAK,IAAI,GAAGI,eAAeJ,KAAK,KAAK,GAAGG,YACxCF,WAAW,KAAK,GAAGG;QAErB,QAAQC,KAAK,GAAG,CACdL,KAAK,GAAG,GAAGM,cAAcN,KAAK,MAAM,GAAGG,YACvCF,WAAW,MAAM,GAAGK;IAExB;IAGA,MAAME,cAAcD,WAAW,KAAK,GAAGA,WAAW,MAAM;IAExD,IAAIC,eAAeN,SACjB,OAAOK;IAIT,MAAME,UAAUF,WAAW,IAAI,GAAGA,WAAW,KAAK,GAAG;IACrD,MAAMG,UAAUH,WAAW,GAAG,GAAGA,WAAW,MAAM,GAAG;IAGrD,MAAMI,cAAcN,KAAK,IAAI,CAACH,UAAUM;IACxC,MAAMI,WAAWP,KAAK,KAAK,CAACE,WAAW,KAAK,GAAGI;IAC/C,MAAME,YAAYR,KAAK,KAAK,CAACE,WAAW,MAAM,GAAGI;IAGjD,MAAMG,UAAUT,KAAK,KAAK,CAACI,UAAUG,WAAW;IAChD,MAAMG,SAASV,KAAK,KAAK,CAACK,UAAUG,YAAY;IAGhD,MAAMG,OAAOX,KAAK,GAAG,CAACS,SAAS;IAC/B,MAAMG,MAAMZ,KAAK,GAAG,CAACU,QAAQ;IAE7B,OAAO;QACLC;QACAC;QACA,OAAOZ,KAAK,GAAG,CAACO,UAAUX,WAAW,KAAK,GAAGe;QAC7C,QAAQX,KAAK,GAAG,CAACQ,WAAWZ,WAAW,MAAM,GAAGgB;IAClD;AACF;AAEO,eAAeC,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcxB,iBAAiBsB,UAAUD,QAAQ,QAAQ;IAE/D,MAAMI,gBAAgB,MAAMC,WAC1BL,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMG,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEF;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaG,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGH,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/section-locate.mjs

@@ -0,0 +1,146 @@
+import { getDebug } from "@aiscene/shared/logger";
+import { assert } from "@aiscene/shared/utils";
+import { userPromptToString } from "../../../common.mjs";
+import { buildSearchAreaLocateSystemPrompt } from "../../prompt/locate/index.mjs";
+import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
+import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
+import { mergePixelBboxesToRect } from "./locate-result-rect.mjs";
+import { buildSearchAreaConfig, expandSearchArea } from "./search-area.mjs";
+import { formatLocateModelContext, prepareLocateModelInput } from "./utils.mjs";
+const debugSection = getDebug('ai:grounding:section');
+async function AiLocateSection(options) {
+    const { context, sectionDescription } = options;
+    const modelRuntime = options.modelRuntime;
+    const { adapter } = modelRuntime;
+    assert('standard' === adapter.locate.kind, 'section locate requires a standard locate adapter');
+    const searchArea = adapter.locate.searchArea;
+    assert(searchArea, 'section locate requires a search area operation');
+    const { protocol: searchAreaProtocol, resultCodec } = searchArea;
+    const screenshotBase64 = context.screenshot.base64;
+    const systemPrompt = buildSearchAreaLocateSystemPrompt({
+        systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,
+        responseInstructions: searchAreaProtocol.buildResponseInstructions(resultCodec.promptSpec)
+    });
+    const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(userPromptToString(sectionDescription));
+    const { messages, preparedImage } = await prepareLocateModelInput({
+        systemPrompt,
+        userPrompt: userInstructionPrompt,
+        locateImage: {
+            imageBase64: screenshotBase64,
+            width: context.shotSize.width,
+            height: context.shotSize.height
+        },
+        imagePreprocess: adapter.imagePreprocess,
+        targetDescription: sectionDescription,
+        userMessageContentOrder: adapter.locate.userMessageContentOrder
+    });
+    let parsedResult;
+    try {
+        parsedResult = await callAiAndParseWithRetry({
+            callAi: (retryAttempt, previousParseError)=>callAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
+                    abortSignal: options.abortSignal,
+                    expectedJsonObjectResponse: searchAreaProtocol.expectedJsonObjectResponse,
+                    semanticRetryAttempt: retryAttempt
+                }),
+            parseResponse: (response)=>{
+                const parsedLocateResult = searchAreaProtocol.parseRawResponse(response.content, resultCodec.promptSpec);
+                const sectionError = parsedLocateResult.error;
+                if ('not-found' === parsedLocateResult.kind) return {
+                    result: response,
+                    sectionError
+                };
+                try {
+                    const locateResultContext = {
+                        preparedSize: preparedImage.preparedSize,
+                        contentSize: preparedImage.contentSize
+                    };
+                    const target = resultCodec.toPixelBbox(parsedLocateResult.target, locateResultContext);
+                    const references = parsedLocateResult.references?.map((reference)=>resultCodec.toPixelBbox(reference, locateResultContext));
+                    const mergedRect = mergePixelBboxesToRect([
+                        target,
+                        ...references ?? []
+                    ]);
+                    debugSection('mergedRect %j', mergedRect);
+                    return {
+                        result: response,
+                        sectionError,
+                        mergedRect
+                    };
+                } catch (error) {
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    const message = [
+                        sectionError && `error in section locate result: ${sectionError}`,
+                        `coordinate parsing error: ${errorMessage}`,
+                        formatLocateModelContext(modelRuntime)
+                    ].filter(Boolean).join('\n');
+                    throw new Error(message, {
+                        cause: error
+                    });
+                }
+            },
+            toParseError: (error, response)=>{
+                const parseErrorMessage = error instanceof Error ? `Failed to parse section locate result: ${error.message}` : 'unknown error in section locate';
+                return new AIResponseParseError(parseErrorMessage, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
+            },
+            parseRetryTimes: modelRuntime.config.retryCount,
+            parseRetryInterval: modelRuntime.config.retryInterval,
+            abortSignal: options.abortSignal,
+            onParseRetry: (error)=>{
+                debugSection('retrying section locate after coordinate parsing failed: %s', error instanceof Error ? error.message : String(error));
+            }
+        });
+    } catch (callError) {
+        if (callError instanceof AIResponseParseError) return {
+            searchAreaConfig: void 0,
+            error: callError.message,
+            rawResponse: callError.rawResponse,
+            rawChoiceMessage: callError.rawChoiceMessage,
+            usage: callError.usage
+        };
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        return {
+            searchAreaConfig: void 0,
+            error: `AI call error: ${errorMessage}`,
+            rawResponse: errorMessage
+        };
+    }
+    const { result, sectionError, mergedRect } = parsedResult;
+    if (!mergedRect) return {
+        searchAreaConfig: void 0,
+        error: sectionError,
+        rawResponse: result.content,
+        rawChoiceMessage: result.rawChoiceMessage,
+        usage: result.usage
+    };
+    try {
+        const expandedRect = expandSearchArea(mergedRect, context.shotSize);
+        const originalWidth = expandedRect.width;
+        const originalHeight = expandedRect.height;
+        debugSection('expanded sectionRect %j', expandedRect);
+        const searchAreaConfig = await buildSearchAreaConfig({
+            context,
+            baseRect: mergedRect
+        });
+        debugSection('scaled section image from %dx%d to %dx%d (scale=%d)', originalWidth, originalHeight, searchAreaConfig.image.width, searchAreaConfig.image.height, searchAreaConfig.mapping.scale);
+        return {
+            searchAreaConfig,
+            error: sectionError,
+            rawResponse: result.content,
+            rawChoiceMessage: result.rawChoiceMessage,
+            usage: result.usage
+        };
+    } catch (error) {
+        const parseErrorMessage = error instanceof Error ? `Failed to parse section locate result: ${error.message}` : 'unknown error in section locate';
+        const errorMessage = sectionError ? `${sectionError} (${parseErrorMessage})` : parseErrorMessage;
+        return {
+            searchAreaConfig: void 0,
+            error: errorMessage,
+            rawResponse: result.content,
+            rawChoiceMessage: result.rawChoiceMessage,
+            usage: result.usage
+        };
+    }
+}
+export { AiLocateSection };
+
+//# sourceMappingURL=section-locate.mjs.map