@midscene/core 0.14.1 → 0.14.2-beta-20250408083201.0

package/dist/types/ai-model.d.ts

@@ -1,8 +1,8 @@
 import { k as AIUsageInfo, Q as PlanningAction } from './types-abcd473f.js';
 import { ChatCompletionMessageParam } from 'openai/resources';
 export { ChatCompletionMessageParam } from 'openai/resources';
-import { b as AIActionType } from './llm-planning-e62b2575.js';
-export { a as AiAssert, e as AiExtractElementInfo, A as AiLocateElement, f as AiLocateSection, g as adaptBboxToRect, c as callAiFn, d as describeUserPage, p as plan, t as transformElementPositionToId } from './llm-planning-e62b2575.js';
+import { b as AIActionType } from './llm-planning-2b762f6e.js';
+export { a as AiAssert, e as AiExtractElementInfo, A as AiLocateElement, f as AiLocateSection, g as adaptBboxToRect, c as callAiFn, d as describeUserPage, p as plan } from './llm-planning-2b762f6e.js';
 import { actionParser } from '@ui-tars/action-parser';
 import '@midscene/shared/constants';
 
@@ -11,7 +11,7 @@ declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[],
     usage?: AIUsageInfo;
 }>;
 
-declare function systemPromptToLocateElement(): "\n## Role:\nYou are an expert in software testing.\n\n## Objective:\n- Identify elements in screenshots and text that match the user's description.\n- Give the coordinates of the element that matches the user's description best in the screenshot.\n\n## Output Format:\n```json\n{\n  \"bbox\": [number, number, number, number],  // top-left x, top-left y, bottom-right x, bottom-right y\n  \"errors\"?: string[]\n}\n```\n\nFields:\n* `bbox` is the bounding box of the element that matches the user's description best in the screenshot\n* `errors` is an optional array of error messages (if any)\n" | "\n## Role:\nYou are an expert in software page image (2D) and page element text analysis.\n\n## Objective:\n- Identify elements in screenshots and text that match the user's description.\n- Return JSON data containing the selection reason and element ID.\n\n## Skills:\n- Image analysis and recognition\n- Multilingual text understanding\n- Software UI design and testing\n\n## Workflow:\n1. Receive the user's element description, screenshot, and element description information. Note that the text may contain non-English characters (e.g., Chinese), indicating that the application may be non-English.\n2. Based on the user's description, locate the target element ID in the list of element descriptions and the screenshot.\n3. Found the required number of elements\n4. Return JSON data containing the selection reason and element ID.\n\n## Constraints:\n- Strictly adhere to the specified location when describing the required element; do not select elements from other locations.\n- Elements in the image with NodeType other than \"TEXT Node\" have been highlighted to identify the element among multiple non-text elements.\n- Accurately identify element information based on the user's description and return the corresponding element ID from the element description information, not extracted from the image.\n- If no elements are found, the \"elements\" array should be empty.\n- The returned data must conform to the specified JSON format.\n- The returned value id information must use the id from element info (important: **use id not indexId, id is hash content**)\n\n## Output Format:\n\nPlease return the result in JSON format as follows:\n\n```json\n{\n  \"elements\": [\n    // If no matching elements are found, return an empty array []\n    {\n      \"reason\": \"PLACEHOLDER\", // The thought process for finding the element, replace PLACEHOLDER with your thought process\n      \"text\": \"PLACEHOLDER\", // Replace PLACEHOLDER with the text of elementInfo, if none, leave empty\n      \"id\": \"PLACEHOLDER\" // Replace PLACEHOLDER with the ID (important: **use id not indexId, id is hash content**) of elementInfo\n    }\n    // More elements...\n  ],\n  \"errors\": [] // Array of strings containing any error messages\n}\n```\n\n## Example:\nExample 1:\nInput Example:\n```json\n// Description: \"Shopping cart icon in the upper right corner\"\n{\n  \"description\": \"PLACEHOLDER\", // Description of the target element\n  \"screenshot\": \"path/screenshot.png\",\n  \"text\": '{\n      \"pageSize\": {\n        \"width\": 400, // Width of the page\n        \"height\": 905 // Height of the page\n      },\n      \"elementInfos\": [\n        {\n          \"id\": \"1231\", // ID of the element\n          \"indexId\": \"0\", // Index of the element，The image is labeled to the left of the element\n          \"attributes\": { // Attributes of the element\n            \"nodeType\": \"IMG Node\", // Type of element, types include: TEXT Node, IMG Node, BUTTON Node, INPUT Node\n            \"src\": \"https://ap-southeast-3.m\",\n            \"class\": \".img\"\n          },\n          \"content\": \"\", // Text content of the element\n          \"rect\": {\n            \"left\": 280, // Distance from the left side of the page\n            \"top\": 8, // Distance from the top of the page\n            \"width\": 44, // Width of the element\n            \"height\": 44 // Height of the element\n          }\n        },\n        {\n          \"id\": \"66551\", // ID of the element\n          \"indexId\": \"1\", // Index of the element,The image is labeled to the left of the element\n          \"attributes\": { // Attributes of the element\n            \"nodeType\": \"IMG Node\", // Type of element, types include: TEXT Node, IMG Node, BUTTON Node, INPUT Node\n            \"src\": \"data:image/png;base64,iVBORw0KGgoAAAANSU...\",\n            \"class\": \".icon\"\n          },\n          \"content\": \"\", // Text content of the element\n          \"rect\": {\n            \"left\": 350, // Distance from the left side of the page\n            \"top\": 16, // Distance from the top of the page\n            \"width\": 25, // Width of the element\n            \"height\": 25 // Height of the element\n          }\n        },\n        ...\n        {\n          \"id\": \"12344\",\n          \"indexId\": \"2\", // Index of the element，The image is labeled to the left of the element\n          \"attributes\": {\n            \"nodeType\": \"TEXT Node\",\n            \"class\": \".product-name\"\n          },\n          \"center\": [\n            288,\n            834\n          ],\n          \"content\": \"Mango Drink\",\n          \"rect\": {\n            \"left\": 188,\n            \"top\": 827,\n            \"width\": 199,\n            \"height\": 13\n          }\n        },\n        ...\n      ]\n    }\n  '\n}\n```\nOutput Example:\n```json\n{\n  \"elements\": [\n    {\n      // Describe the reason for finding this element, replace with actual value in practice\n      \"reason\": \"Reason for finding element 4: It is located in the upper right corner, is an image type, and according to the screenshot, it is a shopping cart icon button\",\n      \"text\": \"\",\n      // ID(**use id not indexId**) of this element, replace with actual value in practice, **use id not indexId**\n      \"id\": \"1231\"\n    }\n  ],\n  \"errors\": []\n}\n```\n  \n  ";
+declare function systemPromptToLocateElement(vlMode: boolean): "\n## Role:\nYou are an expert in software testing.\n\n## Objective:\n- Identify elements in screenshots and text that match the user's description.\n- Give the coordinates of the element that matches the user's description best in the screenshot.\n\n## Output Format:\n```json\n{\n  \"bbox\": [number, number, number, number],  // left, top, right, bottom\n  \"errors\"?: string[]\n}\n```\n\nFields:\n* `bbox` is the bounding box of the element that matches the user's description best in the screenshot\n* `errors` is an optional array of error messages (if any)\n" | "\n## Role:\nYou are an expert in software page image (2D) and page element text analysis.\n\n## Objective:\n- Identify elements in screenshots and text that match the user's description.\n- Return JSON data containing the selection reason and element ID.\n\n## Skills:\n- Image analysis and recognition\n- Multilingual text understanding\n- Software UI design and testing\n\n## Workflow:\n1. Receive the user's element description, screenshot, and element description information. Note that the text may contain non-English characters (e.g., Chinese), indicating that the application may be non-English.\n2. Based on the user's description, locate the target element ID in the list of element descriptions and the screenshot.\n3. Found the required number of elements\n4. Return JSON data containing the selection reason and element ID.\n\n## Constraints:\n- Strictly adhere to the specified location when describing the required element; do not select elements from other locations.\n- Elements in the image with NodeType other than \"TEXT Node\" have been highlighted to identify the element among multiple non-text elements.\n- Accurately identify element information based on the user's description and return the corresponding element ID from the element description information, not extracted from the image.\n- If no elements are found, the \"elements\" array should be empty.\n- The returned data must conform to the specified JSON format.\n- The returned value id information must use the id from element info (important: **use id not indexId, id is hash content**)\n\n## Output Format:\n\nPlease return the result in JSON format as follows:\n\n```json\n{\n  \"elements\": [\n    // If no matching elements are found, return an empty array []\n    {\n      \"reason\": \"PLACEHOLDER\", // The thought process for finding the element, replace PLACEHOLDER with your thought process\n      \"text\": \"PLACEHOLDER\", // Replace PLACEHOLDER with the text of elementInfo, if none, leave empty\n      \"id\": \"PLACEHOLDER\" // Replace PLACEHOLDER with the ID (important: **use id not indexId, id is hash content**) of elementInfo\n    }\n    // More elements...\n  ],\n  \"errors\": [] // Array of strings containing any error messages\n}\n```\n\n## Example:\nExample 1:\nInput Example:\n```json\n// Description: \"Shopping cart icon in the upper right corner\"\n{\n  \"description\": \"PLACEHOLDER\", // Description of the target element\n  \"screenshot\": \"path/screenshot.png\",\n  \"text\": '{\n      \"pageSize\": {\n        \"width\": 400, // Width of the page\n        \"height\": 905 // Height of the page\n      },\n      \"elementInfos\": [\n        {\n          \"id\": \"1231\", // ID of the element\n          \"indexId\": \"0\", // Index of the element，The image is labeled to the left of the element\n          \"attributes\": { // Attributes of the element\n            \"nodeType\": \"IMG Node\", // Type of element, types include: TEXT Node, IMG Node, BUTTON Node, INPUT Node\n            \"src\": \"https://ap-southeast-3.m\",\n            \"class\": \".img\"\n          },\n          \"content\": \"\", // Text content of the element\n          \"rect\": {\n            \"left\": 280, // Distance from the left side of the page\n            \"top\": 8, // Distance from the top of the page\n            \"width\": 44, // Width of the element\n            \"height\": 44 // Height of the element\n          }\n        },\n        {\n          \"id\": \"66551\", // ID of the element\n          \"indexId\": \"1\", // Index of the element,The image is labeled to the left of the element\n          \"attributes\": { // Attributes of the element\n            \"nodeType\": \"IMG Node\", // Type of element, types include: TEXT Node, IMG Node, BUTTON Node, INPUT Node\n            \"src\": \"data:image/png;base64,iVBORw0KGgoAAAANSU...\",\n            \"class\": \".icon\"\n          },\n          \"content\": \"\", // Text content of the element\n          \"rect\": {\n            \"left\": 350, // Distance from the left side of the page\n            \"top\": 16, // Distance from the top of the page\n            \"width\": 25, // Width of the element\n            \"height\": 25 // Height of the element\n          }\n        },\n        ...\n        {\n          \"id\": \"12344\",\n          \"indexId\": \"2\", // Index of the element，The image is labeled to the left of the element\n          \"attributes\": {\n            \"nodeType\": \"TEXT Node\",\n            \"class\": \".product-name\"\n          },\n          \"center\": [\n            288,\n            834\n          ],\n          \"content\": \"Mango Drink\",\n          \"rect\": {\n            \"left\": 188,\n            \"top\": 827,\n            \"width\": 199,\n            \"height\": 13\n          }\n        },\n        ...\n      ]\n    }\n  '\n}\n```\nOutput Example:\n```json\n{\n  \"elements\": [\n    {\n      // Describe the reason for finding this element, replace with actual value in practice\n      \"reason\": \"Reason for finding element 4: It is located in the upper right corner, is an image type, and according to the screenshot, it is a shopping cart icon button\",\n      \"text\": \"\",\n      // ID(**use id not indexId**) of this element, replace with actual value in practice, **use id not indexId**\n      \"id\": \"1231\"\n    }\n  ],\n  \"errors\": []\n}\n```\n  \n  ";
 
 declare function vlmPlanning(options: {
     userInstruction: string;

package/dist/types/index.d.ts

@@ -1,7 +1,7 @@
 import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, B as BaseElement, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightAssertionResponse, A as AISingleElementResponse } from './types-abcd473f.js';
 export { t as AIAssertionResponse, r as AIDataExtractionResponse, p as AIElementCoordinatesResponse, o as AIElementLocatorResponse, q as AIElementResponse, l as AIResponseFormat, s as AISectionLocatorResponse, m as AISingleElementResponseById, n as AISingleElementResponseByPosition, k as AIUsageInfo, K as AgentAssertOpt, J as AgentWaitForOpt, a2 as BaseAgentParserOpt, C as CallAIFn, a1 as Color, x as DumpMeta, H as ElementById, j as ElementTreeNode, u as EnsureObject, a5 as ExecutionRecorderItem, an as ExecutionTaskAction, am as ExecutionTaskActionApply, al as ExecutionTaskInsightAssertion, ak as ExecutionTaskInsightAssertionApply, aj as ExecutionTaskInsightAssertionParam, ac as ExecutionTaskInsightDumpLog, ae as ExecutionTaskInsightLocate, ad as ExecutionTaskInsightLocateApply, ab as ExecutionTaskInsightLocateOutput, aa as ExecutionTaskInsightLocateParam, ai as ExecutionTaskInsightQuery, ah as ExecutionTaskInsightQueryApply, ag as ExecutionTaskInsightQueryOutput, af as ExecutionTaskInsightQueryParam, ap as ExecutionTaskPlanning, ao as ExecutionTaskPlanningApply, a9 as ExecutionTaskReturn, a6 as ExecutionTaskType, a7 as ExecutorContext, aE as FreeFn, aq as GroupedActionDump, z as InsightDump, v as InsightExtractParam, G as LiteUISection, ar as LocateOption, w as LocateResultElement, i as MidsceneYamlFlowItem, au as MidsceneYamlFlowItemAIAction, av as MidsceneYamlFlowItemAIAssert, az as MidsceneYamlFlowItemAIHover, aA as MidsceneYamlFlowItemAIInput, aB as MidsceneYamlFlowItemAIKeyboardPress, aw as MidsceneYamlFlowItemAIQuery, aC as MidsceneYamlFlowItemAIScroll, ay as MidsceneYamlFlowItemAITap, ax as MidsceneYamlFlowItemAIWaitFor, aD as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, at as MidsceneYamlScriptEnv, h as MidsceneYamlTask, O as OnTaskStartTip, F as PartialInsightDumpFromSDK, T as PlanningAIResponse, Q as PlanningAction, Z as PlanningActionParamAssert, $ as PlanningActionParamError, W as PlanningActionParamHover, X as PlanningActionParamInputOrKeyPress, Y as PlanningActionParamScroll, _ as PlanningActionParamSleep, V as PlanningActionParamTap, a0 as PlanningActionParamWaitFor, N as PlanningLocateParam, a4 as PlaywrightParserOpt, P as Point, a3 as PuppeteerParserOpt, R as Rect, y as ReportDumpWithAttributes, aG as ScriptPlayerStatusValue, aF as ScriptPlayerTaskStatus, S as Size, a8 as TaskCacheInfo, as as scrollParam } from './types-abcd473f.js';
-import { c as callAiFn } from './llm-planning-e62b2575.js';
-export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan, t as transformElementPositionToId } from './llm-planning-e62b2575.js';
+import { c as callAiFn } from './llm-planning-2b762f6e.js';
+export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-2b762f6e.js';
 export { getLogDirByType, getVersion, setLogDir } from './utils.js';
 export { MIDSCENE_MODEL_NAME, getAIConfig } from './env.js';
 import '@midscene/shared/constants';

package/dist/types/{llm-planning-e62b2575.d.ts → llm-planning-2b762f6e.d.ts}

@@ -1,4 +1,4 @@
-import { k as AIUsageInfo, R as Rect, B as BaseElement, U as UIContext, q as AIElementResponse, j as ElementTreeNode, o as AIElementLocatorResponse, A as AISingleElementResponse, n as AISingleElementResponseByPosition, H as ElementById, r as AIDataExtractionResponse, t as AIAssertionResponse, T as PlanningAIResponse } from './types-abcd473f.js';
+import { k as AIUsageInfo, R as Rect, B as BaseElement, U as UIContext, A as AISingleElementResponse, n as AISingleElementResponseByPosition, o as AIElementLocatorResponse, H as ElementById, r as AIDataExtractionResponse, t as AIAssertionResponse, T as PlanningAIResponse } from './types-abcd473f.js';
 import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
 
 type AIArgs = [
@@ -40,13 +40,6 @@ declare function describeUserPage<ElementType extends BaseElement = BaseElement>
     };
 }>;
 
-declare function transformElementPositionToId(aiResult: AIElementResponse | [number, number], treeRoot: ElementTreeNode<BaseElement>, size: {
-    width: number;
-    height: number;
-}, searchAreaRect: Rect | undefined, insertElementByPosition: (position: {
-    x: number;
-    y: number;
-}) => BaseElement): Promise<AIElementLocatorResponse>;
 declare function AiLocateElement<ElementType extends BaseElement = BaseElement>(options: {
     context: UIContext<ElementType>;
     targetElementDescription: string;
@@ -94,4 +87,4 @@ declare function plan(userInstruction: string, opts: {
     callAI?: typeof callAiFn<PlanningAIResponse>;
 }): Promise<PlanningAIResponse>;
 
-export { AiLocateElement as A, AiAssert as a, AIActionType as b, callAiFn as c, describeUserPage as d, AiExtractElementInfo as e, AiLocateSection as f, adaptBboxToRect as g, plan as p, transformElementPositionToId as t };
+export { AiLocateElement as A, AiAssert as a, AIActionType as b, callAiFn as c, describeUserPage as d, AiExtractElementInfo as e, AiLocateSection as f, adaptBboxToRect as g, plan as p };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "0.14.1",
+  "version": "0.14.2-beta-20250408083201.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "jsnext:source": "./src/index.ts",
@@ -48,7 +48,7 @@
     "dirty-json": "0.9.2",
     "dotenv": "16.4.5",
     "langsmith": "0.3.7",
-    "@midscene/shared": "0.14.1"
+    "@midscene/shared": "0.14.2-beta-20250408083201.0"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.60.6",