@midscene/core 0.24.1 → 0.24.2-beta-20250730123854.0

package/dist/es/ai-model.d.ts

@@ -1,9 +1,9 @@
-import { ay as StreamingCallback, m as AIUsageInfo, ax as StreamingCodeGenerationOptions, aA as StreamingAIResponse, V as PlanningAction, j as MidsceneYamlFlowItem } from './types-4d9c9a85.js';
+import { aA as StreamingCallback, m as AIUsageInfo, az as StreamingCodeGenerationOptions, aC as StreamingAIResponse, V as PlanningAction, j as MidsceneYamlFlowItem } from './types-d187efec.js';
 import OpenAI from 'openai';
 import { ChatCompletionMessageParam } from 'openai/resources';
 export { ChatCompletionMessageParam } from 'openai/resources';
-import { b as AIActionType } from './llm-planning-3407bd42.js';
-export { a as AiAssert, f as AiExtractElementInfo, A as AiLocateElement, g as AiLocateSection, h as adaptBboxToRect, c as callAiFn, d as describeUserPage, e as elementByPositionWithElementInfo, p as plan } from './llm-planning-3407bd42.js';
+import { b as AIActionType } from './llm-planning-ae19dc49.js';
+export { a as AiAssert, f as AiExtractElementInfo, A as AiLocateElement, g as AiLocateSection, h as adaptBboxToRect, c as callAiFn, d as describeUserPage, e as elementByPositionWithElementInfo, p as plan } from './llm-planning-ae19dc49.js';
 import { vlLocateMode } from '@midscene/shared/env';
 import { actionParser } from '@ui-tars/action-parser';
 import { Size } from '@midscene/shared/types';

package/dist/es/ai-model.js

@@ -18,7 +18,7 @@ import {
   resizeImageForUiTars,
   systemPromptToLocateElement,
   vlmPlanning
-} from "./chunk-TV47VKQ7.js";
+} from "./chunk-3LFZDNMQ.js";
 export {
   AIActionType,
   AiAssert,

package/dist/es/{chunk-TV47VKQ7.js → chunk-3LFZDNMQ.js}

@@ -300,7 +300,7 @@ function buildYamlFlowFromPlans(plans, sleep) {
       flow.push({
         sleep: param.timeMs
       });
-    } else if (type === "AndroidBackButton" || type === "AndroidHomeButton" || type === "AndroidRecentAppsButton") {
+    } else if (type === "AndroidBackButton" || type === "AndroidHomeButton" || type === "AndroidRecentAppsButton" || type === "AndroidLongPress" || type === "AndroidPull") {
     } else if (type === "Error" || type === "ExpectedFalsyCondition" || type === "Assert" || type === "AssertWithoutThrow" || type === "Finished") {
     } else {
       console.warn(
@@ -386,31 +386,47 @@ You are an expert in software testing.
 ## Objective:
 - Identify elements in screenshots and text that match the user's description.
 - Give the coordinates of the element that matches the user's description best in the screenshot.
+- Determine whether the user's description is order-sensitive (e.g., contains phrases like 'the third item in the list', 'the last button', etc.).
 
 ## Output Format:
 \`\`\`json
 {
   "bbox": [number, number, number, number],  // ${bboxComment}
-  "errors"?: string[]
+  "errors"?: string[],
+  "isOrderSensitive": boolean // Whether the targetElementDescription is order-sensitive (true/false)
 }
 \`\`\`
 
 Fields:
 * \`bbox\` is the bounding box of the element that matches the user's description best in the screenshot
+* \`isOrderSensitive\` is a boolean indicating whether the user's description is order-sensitive (true/false)
 * \`errors\` is an optional array of error messages (if any)
 
-For example, when an element is found:
+Order-sensitive means the description contains phrases like:
+- "the third item in the list"
+- "the last button"
+- "the first input box"
+- "the second row"
+
+Not order-sensitive means the description is like:
+- "confirm button"
+- "search box"
+- "password input"
+
+For example, when an element is found and the description is order-sensitive:
 \`\`\`json
 {
   "bbox": [100, 100, 200, 200],
+  "isOrderSensitive": true,
   "errors": []
 }
 \`\`\`
 
-When no element is found:
+When no element is found and the description is not order-sensitive:
 \`\`\`json
 {
   "bbox": [],
+  "isOrderSensitive": false,
   "errors": ["I can see ..., but {some element} is not found"]
 }
 \`\`\`
@@ -423,6 +439,7 @@ You are an expert in software page image (2D) and page element text analysis.
 ## Objective:
 - Identify elements in screenshots and text that match the user's description.
 - Return JSON data containing the selection reason and element ID.
+- Determine whether the user's description is order-sensitive (e.g., contains phrases like 'the third item in the list', 'the last button', etc.).
 
 ## Skills:
 - Image analysis and recognition
@@ -434,6 +451,7 @@ You are an expert in software page image (2D) and page element text analysis.
 2. Based on the user's description, locate the target element ID in the list of element descriptions and the screenshot.
 3. Found the required number of elements
 4. Return JSON data containing the selection reason and element ID.
+5. Judge whether the user's description is order-sensitive (see below for definition and examples).
 
 ## Constraints:
 - Strictly adhere to the specified location when describing the required element; do not select elements from other locations.
@@ -443,6 +461,10 @@ You are an expert in software page image (2D) and page element text analysis.
 - The returned data must conform to the specified JSON format.
 - The returned value id information must use the id from element info (important: **use id not indexId, id is hash content**)
 
+## Order-Sensitive Definition:
+- If the description contains phrases like "the third item in the list", "the last button", "the first input box", "the second row", etc., it is order-sensitive (isOrderSensitive = true).
+- If the description is like "confirm button", "search box", "password input", etc., it is not order-sensitive (isOrderSensitive = false).
+
 ## Output Format:
 
 Please return the result in JSON format as follows:
@@ -458,6 +480,7 @@ Please return the result in JSON format as follows:
     }
     // More elements...
   ],
+  "isOrderSensitive": true, // or false, depending on the user's description
   "errors": [] // Array of strings containing any error messages
 }
 \`\`\`
@@ -546,6 +569,7 @@ Output Example:
       "id": "1231"
     }
   ],
+  "isOrderSensitive": true,
   "errors": []
 }
 \`\`\`
@@ -583,6 +607,10 @@ var locatorSchema = {
           },
           description: "List of found elements"
         },
+        isOrderSensitive: {
+          type: "boolean",
+          description: "Whether the targetElementDescription is order-sensitive (true/false)"
+        },
         errors: {
           type: "array",
           items: {
@@ -591,7 +619,7 @@ var locatorSchema = {
           description: "List of error messages, if any"
         }
       },
-      required: ["elements", "errors"],
+      required: ["elements", "isOrderSensitive", "errors"],
       additionalProperties: false
     }
   }
@@ -766,7 +794,7 @@ Target: User will give you a screenshot, an instruction and some previous logs i
 
 Restriction:
 - Don't give extra actions or plans beyond the instruction. ONLY plan for what the instruction requires. For example, don't try to submit the form if the instruction is only to fill something.
-- Always give ONLY ONE action in \`log\` field (or null if no action should be done), instead of multiple actions. Supported actions are Tap, Hover, Input, KeyboardPress, Scroll${pageType === "android" ? ", AndroidBackButton, AndroidHomeButton, AndroidRecentAppsButton." : "."}
+- Always give ONLY ONE action in \`log\` field (or null if no action should be done), instead of multiple actions. Supported actions are Tap, Hover, Input, KeyboardPress, Scroll${pageType === "android" ? ", AndroidBackButton, AndroidHomeButton, AndroidRecentAppsButton, AndroidLongPress, AndroidPull." : "."}
 - Don't repeat actions in the previous logs.
 - Bbox is the bounding box of the element to be located. It's an array of 4 numbers, representing ${bboxDescription(vlMode)}.
 
@@ -779,7 +807,9 @@ Supporting actions:
 - Scroll: { type: "Scroll", ${vlLocateParam} | null, param: { direction: 'down'(default) | 'up' | 'right' | 'left', scrollType: 'once' (default) | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft', distance: null | number }} // locate is the element to scroll. If it's a page scroll, put \`null\` in the \`locate\` field.
 ${pageType === "android" ? `- AndroidBackButton: { type: "AndroidBackButton", param: {} }
 - AndroidHomeButton: { type: "AndroidHomeButton", param: {} }
-- AndroidRecentAppsButton: { type: "AndroidRecentAppsButton", param: {} }` : ""}
+- AndroidRecentAppsButton: { type: "AndroidRecentAppsButton", param: {} }
+- AndroidLongPress: { type: "AndroidLongPress", param: { x: number, y: number, duration?: number } }
+- AndroidPull: { type: "AndroidPull", param: { direction: 'up' | 'down', startPoint?: { x: number, y: number }, distance?: number, duration?: number } } // Pull down to refresh (direction: 'down') or pull up to load more (direction: 'up')` : ""}
 
 Field description:
 * The \`prompt\` field inside the \`locate\` field is a short description that could be used to locate the element.
@@ -829,7 +859,7 @@ You are a versatile professional in software UI automation. Your outstanding con
 ## Workflow
 
 1. Receive the screenshot, element description of screenshot(if any), user's instruction and previous logs.
-2. Decompose the user's task into a sequence of actions, and place it in the \`actions\` field. There are different types of actions (Tap / Hover / Input / KeyboardPress / Scroll / FalsyConditionStatement / Sleep ${pageType === "android" ? "/ AndroidBackButton / AndroidHomeButton / AndroidRecentAppsButton" : ""}). The "About the action" section below will give you more details.
+2. Decompose the user's task into a sequence of actions, and place it in the \`actions\` field. There are different types of actions (Tap / Hover / Input / KeyboardPress / Scroll / FalsyConditionStatement / Sleep ${pageType === "android" ? "/ AndroidBackButton / AndroidHomeButton / AndroidRecentAppsButton / AndroidLongPress / AndroidPull" : ""}). The "About the action" section below will give you more details.
 3. Precisely locate the target element if it's already shown in the screenshot, put the location info in the \`locate\` field of the action.
 4. If some target elements is not shown in the screenshot, consider the user's instruction is not feasible on this page. Follow the next steps.
 5. Consider whether the user's instruction will be accomplished after all the actions
@@ -888,7 +918,11 @@ ${pageType === "android" ? `- type: 'AndroidBackButton', trigger the system "bac
 - type: 'AndroidHomeButton', trigger the system "home" operation on Android devices
   * {{ param: {{}} }}
 - type: 'AndroidRecentAppsButton', trigger the system "recent apps" operation on Android devices
-  * {{ param: {{}} }}` : ""}
+  * {{ param: {{}} }}
+- type: 'AndroidLongPress', trigger a long press on the screen at specified coordinates on Android devices
+  * {{ param: {{ x: number, y: number, duration?: number }} }}
+- type: 'AndroidPull', trigger pull down to refresh or pull up actions on Android devices
+  * {{ param: {{ direction: 'up' | 'down', startPoint?: {{ x: number, y: number }}, distance?: number, duration?: number }} }}` : ""}
 `;
 var outputTemplate = `
 ## Output JSON Format:
@@ -1002,7 +1036,7 @@ var planSchema = {
               },
               type: {
                 type: "string",
-                description: 'Type of action, one of "Tap", "RightClick", "Hover" , "Input", "KeyboardPress", "Scroll", "ExpectedFalsyCondition", "Sleep", "AndroidBackButton", "AndroidHomeButton", "AndroidRecentAppsButton"'
+                description: 'Type of action, one of "Tap", "RightClick", "Hover" , "Input", "KeyboardPress", "Scroll", "ExpectedFalsyCondition", "Sleep", "AndroidBackButton", "AndroidHomeButton", "AndroidRecentAppsButton", "AndroidLongPress"'
               },
               param: {
                 anyOf: [
@@ -2230,7 +2264,8 @@ async function AiLocateElement(options) {
     },
     rawResponse,
     elementById,
-    usage: res.usage
+    usage: res.usage,
+    isOrderSensitive: typeof res.content === "object" && res.content !== null && "isOrderSensitive" in res.content ? res.content.isOrderSensitive : void 0
   };
 }
 async function AiLocateSection(options) {
@@ -2681,6 +2716,39 @@ async function vlmPlanning(options) {
         type: "AndroidRecentAppsButton",
         param: {}
       });
+    } else if (action.action_type === "androidLongPress") {
+      assert6(
+        action.action_inputs.start_coords,
+        "start_coords is required for androidLongPress"
+      );
+      const point = action.action_inputs.start_coords;
+      transformActions.push({
+        type: "AndroidLongPress",
+        param: {
+          x: point[0],
+          y: point[1],
+          duration: 1e3
+        },
+        locate: null,
+        thought: action.thought || ""
+      });
+    } else if (action.action_type === "androidPull") {
+      const pullDirection = action.action_inputs.direction || "down";
+      const startPoint = action.action_inputs.start_coords ? {
+        x: action.action_inputs.start_coords[0],
+        y: action.action_inputs.start_coords[1]
+      } : void 0;
+      transformActions.push({
+        type: "AndroidPull",
+        param: {
+          direction: pullDirection,
+          startPoint,
+          distance: action.action_inputs.distance,
+          duration: action.action_inputs.duration || 500
+        },
+        locate: null,
+        thought: action.thought || ""
+      });
     }
   });
   if (transformActions.length === 0) {
@@ -2764,4 +2832,4 @@ export {
   resizeImageForUiTars
 };
 
-//# sourceMappingURL=chunk-TV47VKQ7.js.map
+//# sourceMappingURL=chunk-3LFZDNMQ.js.map