@midscene/core 0.30.10 → 1.0.0

package/dist/es/ai-model/conversation-history.mjs

@@ -12,8 +12,10 @@ var _computedKey;
 _computedKey = Symbol.iterator;
 let _computedKey1 = _computedKey;
 class ConversationHistory {
+    resetPendingFeedbackMessageIfExists() {
+        if (this.pendingFeedbackMessage) this.pendingFeedbackMessage = '';
+    }
     append(message) {
-        if ('user' === message.role) this.pruneOldestUserMessageIfNecessary();
         this.messages.push(message);
     }
     seed(messages) {
@@ -25,10 +27,27 @@ class ConversationHistory {
     reset() {
         this.messages.length = 0;
     }
-    snapshot() {
-        return [
+    snapshot(maxImages) {
+        if (void 0 === maxImages) return [
             ...this.messages
         ];
+        const clonedMessages = structuredClone(this.messages);
+        let imageCount = 0;
+        for(let i = clonedMessages.length - 1; i >= 0; i--){
+            const message = clonedMessages[i];
+            const content = message.content;
+            if (Array.isArray(content)) for(let j = 0; j < content.length; j++){
+                const item = content[j];
+                if ('image_url' === item.type) {
+                    imageCount++;
+                    if (imageCount > maxImages) content[j] = {
+                        type: 'text',
+                        text: '(image ignored due to size optimization)'
+                    };
+                }
+            }
+        }
+        return clonedMessages;
     }
     get length() {
         return this.messages.length;
@@ -39,18 +58,11 @@ class ConversationHistory {
     toJSON() {
         return this.snapshot();
     }
-    pruneOldestUserMessageIfNecessary() {
-        const userMessages = this.messages.filter((item)=>'user' === item.role);
-        if (userMessages.length < this.maxUserImageMessages) return;
-        const firstUserMessageIndex = this.messages.findIndex((item)=>'user' === item.role);
-        if (firstUserMessageIndex >= 0) this.messages.splice(firstUserMessageIndex, 1);
-    }
     constructor(options){
-        var _options_initialMessages;
-        _define_property(this, "maxUserImageMessages", void 0);
         _define_property(this, "messages", []);
-        this.maxUserImageMessages = (null == options ? void 0 : options.maxUserImageMessages) ?? 4;
-        if (null == options ? void 0 : null == (_options_initialMessages = options.initialMessages) ? void 0 : _options_initialMessages.length) this.seed(options.initialMessages);
+        _define_property(this, "pendingFeedbackMessage", void 0);
+        if (options?.initialMessages?.length) this.seed(options.initialMessages);
+        this.pendingFeedbackMessage = '';
     }
 }
 export { ConversationHistory };

package/dist/es/ai-model/conversation-history.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/conversation-history.mjs","sources":["webpack://@midscene/core/./src/ai-model/conversation-history.ts"],"sourcesContent":["import type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nexport interface ConversationHistoryOptions {\n  maxUserImageMessages?: number;\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly maxUserImageMessages: number;\n  private readonly messages: ChatCompletionMessageParam[] = [];\n\n  constructor(options?: ConversationHistoryOptions) {\n    this.maxUserImageMessages = options?.maxUserImageMessages ?? 4;\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    if (message.role === 'user') {\n      this.pruneOldestUserMessageIfNecessary();\n    }\n\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n  }\n\n  snapshot(): ChatCompletionMessageParam[] {\n    return [...this.messages];\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n\n  private pruneOldestUserMessageIfNecessary() {\n    const userMessages = this.messages.filter((item) => item.role === 'user');\n    if (userMessages.length < this.maxUserImageMessages) {\n      return;\n    }\n\n    const firstUserMessageIndex = this.messages.findIndex(\n      (item) => item.role === 'user',\n    );\n\n    if (firstUserMessageIndex >= 0) {\n      this.messages.splice(firstUserMessageIndex, 1);\n    }\n  }\n}\n"],"names":["Symbol","ConversationHistory","message","messages","userMessages","item","firstUserMessageIndex","options","_options_initialMessages"],"mappings":";;;;;;;;;;;eA6CGA,OAAO,QAAQ;;AAtCX,MAAMC;IAWX,OAAOC,OAAmC,EAAE;QAC1C,IAAIA,AAAiB,WAAjBA,QAAQ,IAAI,EACd,IAAI,CAAC,iCAAiC;QAGxC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IAEA,WAAyC;QACvC,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACF,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAEQ,oCAAoC;QAC1C,MAAMI,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAACC,OAASA,AAAc,WAAdA,KAAK,IAAI;QAC7D,IAAID,aAAa,MAAM,GAAG,IAAI,CAAC,oBAAoB,EACjD;QAGF,MAAME,wBAAwB,IAAI,CAAC,QAAQ,CAAC,SAAS,CACnD,CAACD,OAASA,AAAc,WAAdA,KAAK,IAAI;QAGrB,IAAIC,yBAAyB,GAC3B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAACA,uBAAuB;IAEhD;IAvDA,YAAYC,OAAoC,CAAE;YAE5CC;QALN,uBAAiB,wBAAjB;QACA,uBAAiB,YAAyC,EAAE;QAG1D,IAAI,CAAC,oBAAoB,GAAGD,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,oBAAoB,AAAD,KAAK;QAC7D,IAAIC,QAAAA,UAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,QAAS,eAAe,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAClC,IAAI,CAAC,IAAI,CAACD,QAAQ,eAAe;IAErC;AAmDF"}
+{"version":3,"file":"ai-model/conversation-history.mjs","sources":["../../../src/ai-model/conversation-history.ts"],"sourcesContent":["import type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nexport interface ConversationHistoryOptions {\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly messages: ChatCompletionMessageParam[] = [];\n\n  public pendingFeedbackMessage: string;\n\n  constructor(options?: ConversationHistoryOptions) {\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n    this.pendingFeedbackMessage = '';\n  }\n\n  resetPendingFeedbackMessageIfExists() {\n    if (this.pendingFeedbackMessage) {\n      this.pendingFeedbackMessage = '';\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n  }\n\n  /**\n   * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n   * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n   * @returns The snapshot of the conversation history.\n   */\n  snapshot(maxImages?: number): ChatCompletionMessageParam[] {\n    if (maxImages === undefined) {\n      return [...this.messages];\n    }\n\n    const clonedMessages = structuredClone(this.messages);\n    let imageCount = 0;\n\n    // Traverse from the end to the beginning\n    for (let i = clonedMessages.length - 1; i >= 0; i--) {\n      const message = clonedMessages[i];\n      const content = message.content;\n\n      // Only process if content is an array\n      if (Array.isArray(content)) {\n        for (let j = 0; j < content.length; j++) {\n          const item = content[j];\n\n          // Check if this is an image\n          if (item.type === 'image_url') {\n            imageCount++;\n\n            // If we've exceeded the limit, replace with text\n            if (imageCount > maxImages) {\n              content[j] = {\n                type: 'text',\n                text: '(image ignored due to size optimization)',\n              };\n            }\n          }\n        }\n      }\n    }\n\n    return clonedMessages;\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n}\n"],"names":["Symbol","ConversationHistory","message","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","content","Array","j","item","options"],"mappings":";;;;;;;;;;;eAqFGA,OAAO,QAAQ;;AA/EX,MAAMC;IAYX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IAOA,SAASE,SAAkB,EAAgC;QACzD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMP,UAAUI,cAAc,CAACG,EAAE;YACjC,MAAMC,UAAUR,QAAQ,OAAO;YAG/B,IAAIS,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,gBAAdA,KAAK,IAAI,EAAkB;oBAC7BL;oBAGA,IAAIA,aAAaJ,WACfM,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAON;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACN,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAhFA,YAAYc,OAAoC,CAAE;QAJlD,uBAAiB,YAAyC,EAAE;QAE5D,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AA4EF"}

package/dist/es/ai-model/index.mjs

@@ -1,11 +1,11 @@
 import { callAI, callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
 import { systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
-import { describeUserPage, elementByPositionWithElementInfo } from "./prompt/util.mjs";
+import { describeUserPage } from "./prompt/util.mjs";
 import { generatePlaywrightTest, generatePlaywrightTestStream } from "./prompt/playwright-generator.mjs";
 import { generateYamlTest, generateYamlTestStream } from "./prompt/yaml-generator.mjs";
-import { AiExtractElementInfo, AiLocateElement, AiLocateSection } from "./inspect.mjs";
+import { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection } from "./inspect.mjs";
 import { plan } from "./llm-planning.mjs";
-import { AIActionType, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, loadActionParam, parseActionParam } from "./common.mjs";
+import { AIActionType, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, loadActionParam, parseActionParam } from "../common.mjs";
 import { resizeImageForUiTars, uiTarsPlanning } from "./ui-tars-planning.mjs";
 import { ConversationHistory } from "./conversation-history.mjs";
-export { AIActionType, AiExtractElementInfo, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, callAI, callAIWithObjectResponse, callAIWithStringResponse, describeUserPage, dumpActionParam, elementByPositionWithElementInfo, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, loadActionParam, parseActionParam, plan, resizeImageForUiTars, systemPromptToLocateElement, uiTarsPlanning };
+export { AIActionType, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, callAI, callAIWithObjectResponse, callAIWithStringResponse, describeUserPage, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, loadActionParam, parseActionParam, plan, resizeImageForUiTars, systemPromptToLocateElement, uiTarsPlanning };

package/dist/es/ai-model/inspect.mjs

@@ -1,11 +1,12 @@
+import { generateElementByPosition } from "@midscene/shared/extractor/dom-util";
 import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
-import { AIActionType, adaptBboxToRect, expandSearchArea, markupImageForLLM, mergeRects } from "./common.mjs";
+import { AIActionType, adaptBboxToRect, expandSearchArea, mergeRects } from "../common.mjs";
 import { extractDataQueryPrompt, systemPromptToExtract } from "./prompt/extraction.mjs";
 import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
-import { describeUserPage, distance, distanceThreshold, elementByPositionWithElementInfo } from "./prompt/util.mjs";
+import { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive } from "./prompt/order-sensitive-judge.mjs";
 import { callAIWithObjectResponse } from "./service-caller/index.mjs";
 const debugInspect = getDebug('ai:inspect');
 const debugSection = getDebug('ai:section');
@@ -14,9 +15,8 @@ const extraTextFromUserPrompt = (prompt)=>{
     return prompt.prompt;
 };
 const promptsToChatParam = async (multimodalPrompt)=>{
-    var _multimodalPrompt_images;
     const msgs = [];
-    if (null == multimodalPrompt ? void 0 : null == (_multimodalPrompt_images = multimodalPrompt.images) ? void 0 : _multimodalPrompt_images.length) {
+    if (multimodalPrompt?.images?.length) {
         msgs.push({
             role: 'user',
             content: [
@@ -33,7 +33,7 @@ const promptsToChatParam = async (multimodalPrompt)=>{
                 content: [
                     {
                         type: 'text',
-                        text: `reference image ${item.name}:`
+                        text: `this is the reference image named '${item.name}':`
                     }
                 ]
             });
@@ -57,14 +57,9 @@ async function AiLocateElement(options) {
     const { context, targetElementDescription, callAIFn, modelConfig } = options;
     const { vlMode } = modelConfig;
     const { screenshotBase64 } = context;
-    const { description, elementById, insertElementByPosition } = await describeUserPage(context, {
-        vlMode
-    });
     assert(targetElementDescription, "cannot find the target element description");
-    const userInstructionPrompt = findElementPrompt({
-        pageDescription: description,
-        targetElementDescription: extraTextFromUserPrompt(targetElementDescription)
-    });
+    const targetElementDescriptionText = extraTextFromUserPrompt(targetElementDescription);
+    const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);
     const systemPrompt = systemPromptToLocateElement(vlMode);
     let imagePayload = screenshotBase64;
     let imageWidth = context.size.width;
@@ -72,21 +67,19 @@ async function AiLocateElement(options) {
     let originalImageWidth = imageWidth;
     let originalImageHeight = imageHeight;
     if (options.searchConfig) {
-        var _options_searchConfig_rect, _options_searchConfig_rect1;
         assert(options.searchConfig.rect, 'searchArea is provided but its rect cannot be found. Failed to locate element');
         assert(options.searchConfig.imageBase64, 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element');
         imagePayload = options.searchConfig.imageBase64;
-        imageWidth = null == (_options_searchConfig_rect = options.searchConfig.rect) ? void 0 : _options_searchConfig_rect.width;
-        imageHeight = null == (_options_searchConfig_rect1 = options.searchConfig.rect) ? void 0 : _options_searchConfig_rect1.height;
+        imageWidth = options.searchConfig.rect?.width;
+        imageHeight = options.searchConfig.rect?.height;
         originalImageWidth = imageWidth;
         originalImageHeight = imageHeight;
-    } else if ('qwen-vl' === vlMode) {
+    } else if ('qwen2.5-vl' === vlMode) {
         const paddedResult = await paddingToMatchBlockByBase64(imagePayload);
         imageWidth = paddedResult.width;
         imageHeight = paddedResult.height;
         imagePayload = paddedResult.imageBase64;
-    } else if ('qwen3-vl' === vlMode) ;
-    else if (!vlMode) imagePayload = await markupImageForLLM(screenshotBase64, context.tree, context.size);
+    }
     const msgs = [
         {
             role: 'system',
@@ -122,30 +115,22 @@ async function AiLocateElement(options) {
     let matchedElements = 'elements' in res.content ? res.content.elements : [];
     let errors = 'errors' in res.content ? res.content.errors : [];
     try {
-        if ('bbox' in res.content && Array.isArray(res.content.bbox)) {
-            var _options_searchConfig_rect2, _options_searchConfig, _options_searchConfig_rect3, _options_searchConfig1;
-            resRect = adaptBboxToRect(res.content.bbox, imageWidth, imageHeight, null == (_options_searchConfig = options.searchConfig) ? void 0 : null == (_options_searchConfig_rect2 = _options_searchConfig.rect) ? void 0 : _options_searchConfig_rect2.left, null == (_options_searchConfig1 = options.searchConfig) ? void 0 : null == (_options_searchConfig_rect3 = _options_searchConfig1.rect) ? void 0 : _options_searchConfig_rect3.top, originalImageWidth, originalImageHeight, vlMode);
+        if ('bbox' in res.content && Array.isArray(res.content.bbox) && res.content.bbox.length >= 1) {
+            resRect = adaptBboxToRect(res.content.bbox, imageWidth, imageHeight, options.searchConfig?.rect?.left, options.searchConfig?.rect?.top, originalImageWidth, originalImageHeight, vlMode);
             debugInspect('resRect', resRect);
             const rectCenter = {
                 x: resRect.left + resRect.width / 2,
                 y: resRect.top + resRect.height / 2
             };
-            let element = elementByPositionWithElementInfo(context.tree, rectCenter);
-            const distanceToCenter = element ? distance({
-                x: element.center[0],
-                y: element.center[1]
-            }, rectCenter) : 0;
-            if (!element || distanceToCenter > distanceThreshold) element = insertElementByPosition(rectCenter);
-            if (element) {
-                matchedElements = [
-                    element
-                ];
-                errors = [];
-            }
+            const element = generateElementByPosition(rectCenter, targetElementDescriptionText);
+            errors = [];
+            if (element) matchedElements = [
+                element
+            ];
         }
     } catch (e) {
         const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
-        if (errors && (null == errors ? void 0 : errors.length) !== 0) errors.push(`(${msg})`);
+        if (errors && errors?.length !== 0) errors.push(`(${msg})`);
         else errors = [
             msg
         ];
@@ -154,12 +139,10 @@ async function AiLocateElement(options) {
         rect: resRect,
         parseResult: {
             elements: matchedElements,
-            errors
+            errors: errors
         },
         rawResponse,
-        elementById,
-        usage: res.usage,
-        isOrderSensitive: 'object' == typeof res.content && null !== res.content && 'isOrderSensitive' in res.content ? res.content.isOrderSensitive : void 0
+        usage: res.usage
     };
 }
 async function AiLocateSection(options) {
@@ -167,9 +150,7 @@ async function AiLocateSection(options) {
     const { vlMode } = modelConfig;
     const { screenshotBase64 } = context;
     const systemPrompt = systemPromptToLocateSection(vlMode);
-    const sectionLocatorInstructionText = sectionLocatorInstruction({
-        sectionDescription: extraTextFromUserPrompt(sectionDescription)
-    });
+    const sectionLocatorInstructionText = sectionLocatorInstruction(extraTextFromUserPrompt(sectionDescription));
     const msgs = [
         {
             role: 'system',
@@ -219,7 +200,7 @@ async function AiLocateSection(options) {
     }
     let imageBase64 = screenshotBase64;
     if (sectionRect) {
-        const croppedResult = await cropByRect(screenshotBase64, sectionRect, 'qwen-vl' === vlMode);
+        const croppedResult = await cropByRect(screenshotBase64, sectionRect, 'qwen2.5-vl' === vlMode);
         imageBase64 = croppedResult.imageBase64;
         sectionRect.width = croppedResult.width;
         sectionRect.height = croppedResult.height;
@@ -234,19 +215,11 @@ async function AiLocateSection(options) {
 }
 async function AiExtractElementInfo(options) {
     const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } = options;
-    const { vlMode } = modelConfig;
     const systemPrompt = systemPromptToExtract();
     const { screenshotBase64 } = context;
-    const { description, elementById } = await describeUserPage(context, {
-        truncateTextLength: 200,
-        filterNonTextContent: false,
-        visibleOnly: false,
-        domIncluded: null == extractOption ? void 0 : extractOption.domIncluded,
-        vlMode
-    });
-    const extractDataPromptText = extractDataQueryPrompt(description, dataQuery);
+    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery);
     const userContent = [];
-    if ((null == extractOption ? void 0 : extractOption.screenshotIncluded) !== false) userContent.push({
+    if (extractOption?.screenshotIncluded !== false) userContent.push({
         type: 'image_url',
         image_url: {
             url: screenshotBase64,
@@ -277,10 +250,28 @@ async function AiExtractElementInfo(options) {
     const result = await callAIWithObjectResponse(msgs, AIActionType.EXTRACT_DATA, modelConfig);
     return {
         parseResult: result.content,
-        elementById,
         usage: result.usage
     };
 }
-export { AiExtractElementInfo, AiLocateElement, AiLocateSection };
+async function AiJudgeOrderSensitive(description, callAIFn, modelConfig) {
+    const systemPrompt = systemPromptToJudgeOrderSensitive();
+    const userPrompt = orderSensitiveJudgePrompt(description);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userPrompt
+        }
+    ];
+    const result = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);
+    return {
+        isOrderSensitive: result.content.isOrderSensitive ?? false,
+        usage: result.usage
+    };
+}
+export { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection };
 
 //# sourceMappingURL=inspect.mjs.map

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocatorResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  BaseElement,\n  ElementById,\n  InsightExtractOption,\n  Rect,\n  ReferenceImage,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from './common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  markupImageForLLM,\n  mergeRects,\n} from './common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  describeUserPage,\n  distance,\n  distanceThreshold,\n  elementByPositionWithElementInfo,\n} from './prompt/util';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `reference image ${item.name}:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement<\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  context: UIContext<ElementType>;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: AIElementLocatorResponse;\n  rect?: Rect;\n  rawResponse: string;\n  elementById: ElementById;\n  usage?: AIUsageInfo;\n  isOrderSensitive?: boolean;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const { description, elementById, insertElementByPosition } =\n    await describeUserPage(context, { vlMode });\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const userInstructionPrompt = findElementPrompt({\n    pageDescription: description,\n    targetElementDescription: extraTextFromUserPrompt(targetElementDescription),\n  });\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  } else if (vlMode === 'qwen3-vl') {\n    // const paddedResult = await paddingToMatchBlockByBase64(imagePayload, 32);\n    // imageWidth = paddedResult.width;\n    // imageHeight = paddedResult.height;\n    // imagePayload = paddedResult.imageBase64;\n  } else if (!vlMode) {\n    imagePayload = await markupImageForLLM(\n      screenshotBase64,\n      context.tree,\n      context.size,\n    );\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements: AIElementLocatorResponse['elements'] =\n    'elements' in res.content ? res.content.elements : [];\n  let errors: AIElementLocatorResponse['errors'] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if ('bbox' in res.content && Array.isArray(res.content.bbox)) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n      let element = elementByPositionWithElementInfo(context.tree, rectCenter);\n\n      const distanceToCenter = element\n        ? distance({ x: element.center[0], y: element.center[1] }, rectCenter)\n        : 0;\n\n      if (!element || distanceToCenter > distanceThreshold) {\n        element = insertElementByPosition(rectCenter);\n      }\n\n      if (element) {\n        matchedElements = [element];\n        errors = [];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements,\n      errors,\n    },\n    rawResponse,\n    elementById,\n    usage: res.usage,\n    isOrderSensitive:\n      typeof res.content === 'object' &&\n      res.content !== null &&\n      'isOrderSensitive' in res.content\n        ? (res.content as any).isOrderSensitive\n        : undefined,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext<BaseElement>;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = sectionLocatorInstruction({\n    sectionDescription: extraTextFromUserPrompt(sectionDescription),\n  });\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<\n  T,\n  ElementType extends BaseElement = BaseElement,\n>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext<ElementType>;\n  extractOption?: InsightExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const { vlMode } = modelConfig;\n  const systemPrompt = systemPromptToExtract();\n\n  const { screenshotBase64 } = context;\n\n  const { description, elementById } = await describeUserPage(context, {\n    truncateTextLength: 200,\n    filterNonTextContent: false,\n    visibleOnly: false,\n    domIncluded: extractOption?.domIncluded,\n    vlMode,\n  });\n\n  const extractDataPromptText = extractDataQueryPrompt(description, dataQuery);\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    elementById,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","description","elementById","insertElementByPosition","describeUserPage","assert","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","_options_searchConfig_rect","_options_searchConfig_rect1","paddedResult","paddingToMatchBlockByBase64","markupImageForLLM","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","elementByPositionWithElementInfo","distanceToCenter","distance","distanceThreshold","e","msg","Error","undefined","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent"],"mappings":";;;;;;;;;AA0DA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,gBAAgB,EAAEC,KAAK,IAAI,CAAC,CAAC,CAAC;oBACvC;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAEpBC,OASD;IAQC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAM,EAAEM,WAAW,EAAEC,WAAW,EAAEC,uBAAuB,EAAE,GACzD,MAAMC,iBAAiBT,SAAS;QAAEI;IAAO;IAE3CM,OACET,0BACA;IAEF,MAAMU,wBAAwBC,kBAAkB;QAC9C,iBAAiBN;QACjB,0BAA0BjB,wBAAwBY;IACpD;IACA,MAAMY,eAAeC,4BAA4BV;IAEjD,IAAIW,eAAeV;IACnB,IAAIW,aAAahB,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIiB,cAAcjB,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIkB,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIlB,QAAQ,YAAY,EAAE;YAWXqB,4BACCC;QAXdX,OACEX,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFW,OACEX,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFgB,eAAehB,QAAQ,YAAY,CAAC,WAAW;QAC/CiB,aAAa,QAAAI,CAAAA,6BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,KAAK;QAC7CH,cAAc,QAAAI,CAAAA,8BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,MAAM;QAC/CH,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIb,AAAW,cAAXA,QAAsB;QAC/B,MAAMkB,eAAe,MAAMC,4BAA4BR;QACvDC,aAAaM,aAAa,KAAK;QAC/BL,cAAcK,aAAa,MAAM;QACjCP,eAAeO,aAAa,WAAW;IACzC,OAAO,IAAIlB,AAAW,eAAXA;SAKJ,IAAI,CAACA,QACVW,eAAe,MAAMS,kBACnBnB,kBACAL,QAAQ,IAAI,EACZA,QAAQ,IAAI;IAIhB,MAAMN,OAAe;QACnB;YAAE,MAAM;YAAU,SAASmB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMJ;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOV,0BAAuC;QAChD,MAAMwB,SAAS,MAAMlC,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI+B;IACf;IAEA,MAAMC,MAAM,MAAMxB,SAASR,MAAMiC,aAAa,eAAe,EAAExB;IAE/D,MAAMyB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBACF,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IACvD,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IAAI,UAAUA,IAAI,OAAO,IAAIO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,GAAG;gBAK1DQ,6BAAAA,uBACAC,6BAAAA;YALFL,UAAUM,gBACRV,IAAI,OAAO,CAAC,IAAI,EAChBV,YACAC,aAAAA,QACAiB,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG,EAC/BjB,oBACAC,qBACAf;YAGFlB,aAAa,WAAW4C;YAExB,MAAMO,aAAa;gBACjB,GAAGP,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YACA,IAAIQ,UAAUC,iCAAiCvC,QAAQ,IAAI,EAAEqC;YAE7D,MAAMG,mBAAmBF,UACrBG,SAAS;gBAAE,GAAGH,QAAQ,MAAM,CAAC,EAAE;gBAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;YAAC,GAAGD,cACzD;YAEJ,IAAI,CAACC,WAAWE,mBAAmBE,mBACjCJ,UAAU9B,wBAAwB6B;YAGpC,IAAIC,SAAS;gBACXP,kBAAkB;oBAACO;iBAAQ;gBAC3BN,SAAS,EAAE;YACb;QACF;IACF,EAAE,OAAOW,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACX,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEY,IAAI,CAAC,CAAC;aAFtBZ,SAAS;YAACY;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMd;QACN,aAAa;YACX,UAAUC;YACVC;QACF;QACAJ;QACArB;QACA,OAAOmB,IAAI,KAAK;QAChB,kBACE,AAAuB,YAAvB,OAAOA,IAAI,OAAO,IAClBA,AAAgB,SAAhBA,IAAI,OAAO,IACX,sBAAsBA,IAAI,OAAO,GAC5BA,IAAI,OAAO,CAAS,gBAAgB,GACrCoB;IACR;AACF;AAEO,eAAeC,gBAAgBhD,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEgD,kBAAkB,EAAE7C,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMa,eAAeoC,4BAA4B7C;IACjD,MAAM8C,gCAAgCC,0BAA0B;QAC9D,oBAAoB9D,wBAAwB2D;IAC9C;IACA,MAAMtD,OAAe;QACnB;YAAE,MAAM;YAAU,SAASmB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKR;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM6C;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMvB,SAAS,MAAMlC,mBAAmB;YACtC,QAAQyD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACAtD,KAAK,IAAI,IAAI+B;IACf;IAEA,MAAM2B,SAAS,MAAMC,yBACnB3D,MACAiC,aAAa,YAAY,EACzBxB;IAGF,IAAImD;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAapB,gBACjBmB,aACAvD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0BoE;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DhE,aAAa,wBAAwBqE;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAS1B,MAAM,OAAO,CAAC0B,OAC/B,GAAG,CAAC,CAACA,OACGvB,gBACLuB,MACA3D,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqBsE;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DtE,aAAa,iBAAiBwE;QAG9BN,cAAcQ,iBAAiBF,YAAY5D,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2BkE;IAC1C;IAEA,IAAIS,cAAc1D;IAClB,IAAIiD,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1B5D,kBACAiD,aACAlD,AAAW,cAAXA;QAEF2D,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAavB,KAAK,SAAS,CAACuB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAGpBnE,OAMD;IACC,MAAM,EAAEoE,SAAS,EAAEnE,OAAO,EAAEoE,aAAa,EAAE5E,gBAAgB,EAAEW,WAAW,EAAE,GACxEJ;IACF,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAMU,eAAewD;IAErB,MAAM,EAAEhE,gBAAgB,EAAE,GAAGL;IAE7B,MAAM,EAAEM,WAAW,EAAEC,WAAW,EAAE,GAAG,MAAME,iBAAiBT,SAAS;QACnE,oBAAoB;QACpB,sBAAsB;QACtB,aAAa;QACb,aAAaoE,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,WAAW;QACvChE;IACF;IAEA,MAAMkE,wBAAwBC,uBAAuBjE,aAAa6D;IAElE,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKnE;YACL,QAAQ;QACV;IACF;IAGFmE,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM5E,OAAe;QACnB;YAAE,MAAM;YAAU,SAASmB;QAAa;QACxC;YACE,MAAM;YACN,SAAS2D;QACX;KACD;IAED,IAAIhF,kBAAkB;QACpB,MAAMiC,SAAS,MAAMlC,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAI+B;IACf;IAEA,MAAM2B,SAAS,MAAMC,yBACnB3D,MACAiC,aAAa,YAAY,EACzBxB;IAEF,OAAO;QACL,aAAaiD,OAAO,OAAO;QAC3B7C;QACA,OAAO6C,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ReferenceImage,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  mergeRects,\n} from '../common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}':`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const targetElementDescriptionText = extraTextFromUserPrompt(\n    targetElementDescription,\n  );\n  const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements = 'elements' in res.content ? res.content.elements : [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if (\n      'bbox' in res.content &&\n      Array.isArray(res.content.bbox) &&\n      res.content.bbox.length >= 1\n    ) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n      errors = [];\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen2.5-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n  modelConfig: IModelConfig,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  const result = await callAIFn(\n    msgs,\n    AIActionType.INSPECT_ELEMENT, // Reuse existing action type for now\n    modelConfig,\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","paddedResult","paddingToMatchBlockByBase64","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","adaptBboxToRect","rectCenter","element","generateElementByPosition","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;AAsDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IASC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,+BAA+BjB,wBACnCW;IAEF,MAAMO,wBAAwBC,kBAAkBF;IAChD,MAAMG,eAAeC,4BAA4BP;IAEjD,IAAIQ,eAAeP;IACnB,IAAIQ,aAAab,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIc,cAAcd,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIe,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIf,QAAQ,YAAY,EAAE;QACxBO,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFa,eAAeb,QAAQ,YAAY,CAAC,WAAW;QAC/Cc,aAAad,QAAQ,YAAY,CAAC,IAAI,EAAE;QACxCe,cAAcf,QAAQ,YAAY,CAAC,IAAI,EAAE;QACzCgB,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIV,AAAW,iBAAXA,QAAyB;QAClC,MAAMa,eAAe,MAAMC,4BAA4BN;QACvDC,aAAaI,aAAa,KAAK;QAC/BH,cAAcG,aAAa,MAAM;QACjCL,eAAeK,aAAa,WAAW;IACzC;IAEA,MAAMvB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMJ;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOP,0BAAuC;QAChD,MAAMkB,SAAS,MAAM3B,mBAAmB;YACtC,QAAQS,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMC,MAAM,MAAMlB,SAASR,MAAM2B,aAAa,eAAe,EAAElB;IAE/D,MAAMmB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBAAkB,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IAC3E,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;YACAI,UAAUI,gBACRR,IAAI,OAAO,CAAC,IAAI,EAChBP,YACAC,aACAf,QAAQ,YAAY,EAAE,MAAM,MAC5BA,QAAQ,YAAY,EAAE,MAAM,KAC5BgB,oBACAC,qBACAZ;YAGFjB,aAAa,WAAWqC;YAExB,MAAMK,aAAa;gBACjB,GAAGL,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMM,UAA+BC,0BACnCF,YACAtB;YAEFmB,SAAS,EAAE;YAEX,IAAII,SACFL,kBAAkB;gBAACK;aAAQ;QAE/B;IACF,EAAE,OAAOE,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACN,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;aAFtBP,SAAS;YAACO;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMT;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAJ;QACA,OAAOF,IAAI,KAAK;IAClB;AACF;AAEO,eAAee,gBAAgBpC,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEoC,kBAAkB,EAAEjC,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAe2B,4BAA4BjC;IACjD,MAAMkC,gCAAgCC,0BACpCjD,wBAAwB8C;IAE1B,MAAM1C,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMiC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMjB,SAAS,MAAM3B,mBAAmB;YACtC,QAAQ4C,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA1C,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMqB,SAAS,MAAMC,yBACnB/C,MACA2B,aAAa,YAAY,EACzBlB;IAGF,IAAIuC;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAahB,gBACjBe,aACA3C,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFf,aAAa,0BAA0BuD;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DnD,aAAa,wBAAwBwD;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAASpB,MAAM,OAAO,CAACoB,OAC/B,GAAG,CAAC,CAACA,OACGnB,gBACLmB,MACA/C,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNf,aAAa,qBAAqByD;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7DzD,aAAa,iBAAiB2D;QAG9BN,cAAcQ,iBAAiBF,YAAYhD,QAAQ,IAAI,EAAEI;QACzDf,aAAa,2BAA2BqD;IAC1C;IAEA,IAAIS,cAAc9C;IAClB,IAAIqC,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1BhD,kBACAqC,aACAtC,AAAW,iBAAXA;QAEF+C,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAajB,KAAK,SAAS,CAACiB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwBvD,OAO7C;IACC,MAAM,EAAEwD,SAAS,EAAEvD,OAAO,EAAEwD,aAAa,EAAE/D,gBAAgB,EAAEU,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAe+C;IACrB,MAAM,EAAEpD,gBAAgB,EAAE,GAAGL;IAE7B,MAAM0D,wBAAwBC,uBAC5B5D,QAAQ,eAAe,IAAI,IAC3BwD;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAKvD;YACL,QAAQ;QACV;IACF;IAGFuD,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMhE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASkD;QACX;KACD;IAED,IAAInE,kBAAkB;QACpB,MAAM0B,SAAS,MAAM3B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAIyB;IACf;IAEA,MAAMqB,SAAS,MAAMC,yBACnB/C,MACA2B,aAAa,YAAY,EACzBlB;IAEF,OAAO;QACL,aAAaqC,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeqB,sBACpBC,WAAmB,EACnB5D,QAAwE,EACxEC,WAAyB;IAKzB,MAAMO,eAAeqD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAMpE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASsD;QACX;KACD;IAED,MAAMxB,SAAS,MAAMtC,SACnBR,MACA2B,aAAa,eAAe,EAC5BlB;IAGF,OAAO;QACL,kBAAkBqC,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/llm-planning.mjs

@@ -1,95 +1,91 @@
 import { paddingToMatchBlockByBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
-import { AIActionType, buildYamlFlowFromPlans, fillBboxParam, findAllMidsceneLocatorField, markupImageForLLM, warnGPT4oSizeLimit } from "./common.mjs";
+import { AIActionType, buildYamlFlowFromPlans, fillBboxParam, findAllMidsceneLocatorField } from "../common.mjs";
 import { systemPromptToTaskPlanning } from "./prompt/llm-planning.mjs";
-import { describeUserPage } from "./prompt/util.mjs";
 import { callAIWithObjectResponse } from "./service-caller/index.mjs";
 const debug = getDebug('planning');
 async function plan(userInstruction, opts) {
-    var _opts_conversationHistory, _planFromAI_action;
     const { context, modelConfig, conversationHistory } = opts;
     const { screenshotBase64, size } = context;
-    const { modelName, vlMode } = modelConfig;
-    const { description: pageDescription, elementById } = await describeUserPage(context, {
-        vlMode
-    });
+    const { vlMode } = modelConfig;
     const systemPrompt = await systemPromptToTaskPlanning({
         actionSpace: opts.actionSpace,
-        vlMode: vlMode
+        vlMode,
+        includeBbox: opts.includeBbox
     });
     let imagePayload = screenshotBase64;
     let imageWidth = size.width;
     let imageHeight = size.height;
     const rightLimit = imageWidth;
     const bottomLimit = imageHeight;
-    if ('qwen-vl' === vlMode) {
+    if ('qwen2.5-vl' === vlMode) {
         const paddedResult = await paddingToMatchBlockByBase64(imagePayload);
         imageWidth = paddedResult.width;
         imageHeight = paddedResult.height;
         imagePayload = paddedResult.imageBase64;
-    } else if ('qwen3-vl' === vlMode) ;
-    else if (!vlMode) imagePayload = await markupImageForLLM(screenshotBase64, context.tree, {
-        width: imageWidth,
-        height: imageHeight
-    });
-    warnGPT4oSizeLimit(size, modelName);
-    const historyLog = (null == (_opts_conversationHistory = opts.conversationHistory) ? void 0 : _opts_conversationHistory.snapshot()) || [];
-    const knowledgeContext = opts.actionContext ? [
+    }
+    const actionContext = opts.actionContext ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\n` : '';
+    const instruction = [
         {
             role: 'user',
             content: [
                 {
                     type: 'text',
-                    text: `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>`
+                    text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`
                 }
             ]
         }
-    ] : [];
-    const instruction = [
-        {
+    ];
+    let latestFeedbackMessage;
+    if (conversationHistory.pendingFeedbackMessage) {
+        latestFeedbackMessage = {
             role: 'user',
             content: [
                 {
                     type: 'text',
-                    text: `<user_instruction>${userInstruction}</user_instruction>`
+                    text: `${conversationHistory.pendingFeedbackMessage}. The last screenshot is attached. Please going on according to the instruction.`
+                },
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
                 }
             ]
-        }
-    ];
+        };
+        conversationHistory.resetPendingFeedbackMessageIfExists();
+    } else latestFeedbackMessage = {
+        role: 'user',
+        content: [
+            {
+                type: 'text',
+                text: 'this is the latest screenshot'
+            },
+            {
+                type: 'image_url',
+                image_url: {
+                    url: imagePayload,
+                    detail: 'high'
+                }
+            }
+        ]
+    };
+    conversationHistory.append(latestFeedbackMessage);
+    const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);
     const msgs = [
         {
             role: 'system',
             content: systemPrompt
         },
-        ...knowledgeContext,
         ...instruction,
-        ...historyLog,
-        {
-            role: 'user',
-            content: [
-                {
-                    type: 'image_url',
-                    image_url: {
-                        url: imagePayload,
-                        detail: 'high'
-                    }
-                },
-                ...vlMode ? [] : [
-                    {
-                        type: 'text',
-                        text: pageDescription
-                    }
-                ]
-            ]
-        }
+        ...historyLog
     ];
-    const { content, usage } = await callAIWithObjectResponse(msgs, AIActionType.PLAN, modelConfig);
-    const rawResponse = JSON.stringify(content, void 0, 2);
-    const planFromAI = content;
-    const actions = ((null == (_planFromAI_action = planFromAI.action) ? void 0 : _planFromAI_action.type) ? [
+    const { content: planFromAI, contentString: rawResponse, usage } = await callAIWithObjectResponse(msgs, AIActionType.PLAN, modelConfig);
+    const actions = planFromAI.action ? [
         planFromAI.action
-    ] : planFromAI.actions) || [];
+    ] : [];
     const returnValue = {
         ...planFromAI,
         actions,
@@ -106,16 +102,11 @@ async function plan(userInstruction, opts) {
         debug('locateFields', locateFields);
         locateFields.forEach((field)=>{
             const locateResult = action.param[field];
-            if (locateResult) if (vlMode) action.param[field] = fillBboxParam(locateResult, imageWidth, imageHeight, rightLimit, bottomLimit, vlMode);
-            else {
-                const element = elementById(locateResult);
-                if (element) action.param[field].id = element.id;
-            }
+            if (locateResult && void 0 !== vlMode) action.param[field] = fillBboxParam(locateResult, imageWidth, imageHeight, rightLimit, bottomLimit, vlMode);
         });
     });
-    assert(!planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);
     if (0 === actions.length && returnValue.more_actions_needed_by_instruction && !returnValue.sleep) console.warn('No actions planned for the prompt, but model said more actions are needed:', userInstruction);
-    null == conversationHistory || conversationHistory.append({
+    conversationHistory.append({
         role: 'assistant',
         content: [
             {
@@ -124,15 +115,6 @@ async function plan(userInstruction, opts) {
             }
         ]
     });
-    null == conversationHistory || conversationHistory.append({
-        role: 'user',
-        content: [
-            {
-                type: 'text',
-                text: 'I have finished the action previously planned'
-            }
-        ]
-    });
     return returnValue;
 }
 export { plan };