@midscene/core 0.24.1-beta-20250729062523.0 → 0.24.1-beta-20250729081015.0

package/dist/es/ai-model.d.ts

@@ -1,9 +1,9 @@
-import { ay as StreamingCallback, m as AIUsageInfo, ax as StreamingCodeGenerationOptions, aA as StreamingAIResponse, V as PlanningAction, j as MidsceneYamlFlowItem } from './types-4d9c9a85.js';
+import { ay as StreamingCallback, m as AIUsageInfo, ax as StreamingCodeGenerationOptions, aA as StreamingAIResponse, V as PlanningAction, j as MidsceneYamlFlowItem } from './types-8d202846.js';
 import OpenAI from 'openai';
 import { ChatCompletionMessageParam } from 'openai/resources';
 export { ChatCompletionMessageParam } from 'openai/resources';
-import { b as AIActionType } from './llm-planning-3407bd42.js';
-export { a as AiAssert, f as AiExtractElementInfo, A as AiLocateElement, g as AiLocateSection, h as adaptBboxToRect, c as callAiFn, d as describeUserPage, e as elementByPositionWithElementInfo, p as plan } from './llm-planning-3407bd42.js';
+import { b as AIActionType } from './llm-planning-7d13d5ff.js';
+export { a as AiAssert, f as AiExtractElementInfo, A as AiLocateElement, g as AiLocateSection, h as adaptBboxToRect, c as callAiFn, d as describeUserPage, e as elementByPositionWithElementInfo, p as plan } from './llm-planning-7d13d5ff.js';
 import { vlLocateMode } from '@midscene/shared/env';
 import { actionParser } from '@ui-tars/action-parser';
 import { Size } from '@midscene/shared/types';

package/dist/es/ai-model.js

@@ -18,7 +18,7 @@ import {
   resizeImageForUiTars,
   systemPromptToLocateElement,
   vlmPlanning
-} from "./chunk-TV47VKQ7.js";
+} from "./chunk-GPQNKTL4.js";
 export {
   AIActionType,
   AiAssert,

package/dist/es/{chunk-TV47VKQ7.js → chunk-GPQNKTL4.js}

@@ -386,31 +386,47 @@ You are an expert in software testing.
 ## Objective:
 - Identify elements in screenshots and text that match the user's description.
 - Give the coordinates of the element that matches the user's description best in the screenshot.
+- Determine whether the user's description is order-sensitive (e.g., contains phrases like 'the third item in the list', 'the last button', etc.).
 
 ## Output Format:
 \`\`\`json
 {
   "bbox": [number, number, number, number],  // ${bboxComment}
-  "errors"?: string[]
+  "errors"?: string[],
+  "isOrderSensitive": boolean // Whether the targetElementDescription is order-sensitive (true/false)
 }
 \`\`\`
 
 Fields:
 * \`bbox\` is the bounding box of the element that matches the user's description best in the screenshot
+* \`isOrderSensitive\` is a boolean indicating whether the user's description is order-sensitive (true/false)
 * \`errors\` is an optional array of error messages (if any)
 
-For example, when an element is found:
+Order-sensitive means the description contains phrases like:
+- "the third item in the list"
+- "the last button"
+- "the first input box"
+- "the second row"
+
+Not order-sensitive means the description is like:
+- "confirm button"
+- "search box"
+- "password input"
+
+For example, when an element is found and the description is order-sensitive:
 \`\`\`json
 {
   "bbox": [100, 100, 200, 200],
+  "isOrderSensitive": true,
   "errors": []
 }
 \`\`\`
 
-When no element is found:
+When no element is found and the description is not order-sensitive:
 \`\`\`json
 {
   "bbox": [],
+  "isOrderSensitive": false,
   "errors": ["I can see ..., but {some element} is not found"]
 }
 \`\`\`
@@ -423,6 +439,7 @@ You are an expert in software page image (2D) and page element text analysis.
 ## Objective:
 - Identify elements in screenshots and text that match the user's description.
 - Return JSON data containing the selection reason and element ID.
+- Determine whether the user's description is order-sensitive (e.g., contains phrases like 'the third item in the list', 'the last button', etc.).
 
 ## Skills:
 - Image analysis and recognition
@@ -434,6 +451,7 @@ You are an expert in software page image (2D) and page element text analysis.
 2. Based on the user's description, locate the target element ID in the list of element descriptions and the screenshot.
 3. Found the required number of elements
 4. Return JSON data containing the selection reason and element ID.
+5. Judge whether the user's description is order-sensitive (see below for definition and examples).
 
 ## Constraints:
 - Strictly adhere to the specified location when describing the required element; do not select elements from other locations.
@@ -443,6 +461,10 @@ You are an expert in software page image (2D) and page element text analysis.
 - The returned data must conform to the specified JSON format.
 - The returned value id information must use the id from element info (important: **use id not indexId, id is hash content**)
 
+## Order-Sensitive Definition:
+- If the description contains phrases like "the third item in the list", "the last button", "the first input box", "the second row", etc., it is order-sensitive (isOrderSensitive = true).
+- If the description is like "confirm button", "search box", "password input", etc., it is not order-sensitive (isOrderSensitive = false).
+
 ## Output Format:
 
 Please return the result in JSON format as follows:
@@ -458,6 +480,7 @@ Please return the result in JSON format as follows:
     }
     // More elements...
   ],
+  "isOrderSensitive": true, // or false, depending on the user's description
   "errors": [] // Array of strings containing any error messages
 }
 \`\`\`
@@ -546,6 +569,7 @@ Output Example:
       "id": "1231"
     }
   ],
+  "isOrderSensitive": true,
   "errors": []
 }
 \`\`\`
@@ -583,6 +607,10 @@ var locatorSchema = {
           },
           description: "List of found elements"
         },
+        isOrderSensitive: {
+          type: "boolean",
+          description: "Whether the targetElementDescription is order-sensitive (true/false)"
+        },
         errors: {
           type: "array",
           items: {
@@ -591,7 +619,7 @@ var locatorSchema = {
           description: "List of error messages, if any"
         }
       },
-      required: ["elements", "errors"],
+      required: ["elements", "isOrderSensitive", "errors"],
       additionalProperties: false
     }
   }
@@ -789,6 +817,7 @@ Return in JSON format:
   ${vlCoTLog}
   ${vlCurrentLog}
   ${commonOutputFields}
+  "isOrderSensitive": boolean, // Whether the user's instruction is order-sensitive (true/false)
   "action": 
     {
       // one of the supporting actions
@@ -901,6 +930,7 @@ The JSON format is as follows:
   ],
   ${llmCurrentLog}
   ${commonOutputFields}
+  "isOrderSensitive": boolean, // Whether the user's instruction is order-sensitive (true/false)
 }}
 
 ## Examples
@@ -1060,6 +1090,10 @@ var planSchema = {
           },
           description: "List of actions to be performed"
         },
+        isOrderSensitive: {
+          type: "boolean",
+          description: "Whether the user's instruction is order-sensitive (true/false)"
+        },
         more_actions_needed_by_instruction: {
           type: "boolean",
           description: "If all the actions described in the instruction have been covered by this action and logs, set this field to false."
@@ -2083,6 +2117,7 @@ import { PromptTemplate as PromptTemplate4 } from "@langchain/core/prompts";
 function systemPromptToLocateSection(vlMode) {
   return `
 You goal is to find out one section containing the target element in the screenshot, put it in the \`bbox\` field. If the user describe the target element with some reference elements, you should also find the section containing the reference elements, put it in the \`references_bbox\` field.
+- Determine whether the user's description is order-sensitive (e.g., contains phrases like 'the third item in the list', 'the last button', etc.).
 
 Usually, it should be approximately an area not more than 300x300px. Changes of the size are allowed if there are many elements to cover.
 
@@ -2095,11 +2130,27 @@ return in this JSON format:
     [number, number, number, number],
     ...
   ],
+  "isOrderSensitive": boolean, // Whether the targetElementDescription is order-sensitive (true/false)
   "error"?: string
 }
 \`\`\`
 
-In which, all the numbers in the \`bbox\` and \`references_bbox\` represent ${bboxDescription(vlMode)}.
+In which, all the numbers in the \`bbox\` and \`references_bbox\` represent ${bboxDescription(
+    vlMode
+  )}.
+
+* \`isOrderSensitive\` is a boolean indicating whether the user's description is order-sensitive (true/false)
+
+Order-sensitive means the description contains phrases like:
+- "the third item in the list"
+- "the last button"
+- "the first input box"
+- "the second row"
+
+Not order-sensitive means the description is like:
+- "confirm button"
+- "search box"
+- "password input"
 
 For example, if the user describe the target element as "the delete button on the second row with title 'Peter'", you should put the bounding box of the delete button in the \`bbox\` field, and the bounding box of the second row in the \`references_bbox\` field.
 
@@ -2107,7 +2158,8 @@ the return value should be like this:
 \`\`\`json
 {
   "bbox": [100, 100, 200, 200],
-  "references_bbox": [[100, 100, 200, 200]]
+  "references_bbox": [[100, 100, 200, 200]],
+  "isOrderSensitive": true
 }
 \`\`\`
 `;
@@ -2230,7 +2282,8 @@ async function AiLocateElement(options) {
     },
     rawResponse,
     elementById,
-    usage: res.usage
+    usage: res.usage,
+    isOrderSensitive: typeof res.content === "object" && res.content !== null && "isOrderSensitive" in res.content ? res.content.isOrderSensitive : void 0
   };
 }
 async function AiLocateSection(options) {
@@ -2764,4 +2817,4 @@ export {
   resizeImageForUiTars
 };
 
-//# sourceMappingURL=chunk-TV47VKQ7.js.map
+//# sourceMappingURL=chunk-GPQNKTL4.js.map