@midscene/core 1.10.2-beta-20260706032158.0 → 1.10.3

package/dist/es/ai-model/prompt/describe.mjs

@@ -21,53 +21,50 @@ const getExamples = (language)=>{
     const examples = examplesMap[language] || examplesMap.English;
     return examples.map((e)=>`- ${e}`).join('\n');
 };
+const describeElementReturnJsonSchema = ()=>`{
+  "description": "unique element identifier",
+  "error"?: "error message if any"
+}`;
 const elementDescriberInstruction = ()=>{
     const preferredLanguage = getPreferredLanguage();
     return `
-Describe the element in the red rectangle for precise identification.
+Describe the real page element indicated by the temporary callout.
+The callout is an annotation overlay. It is not part of the page or target.
+The description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption.
 
 IMPORTANT: You MUST write the description in ${preferredLanguage}.
 
-CRITICAL REQUIREMENTS:
-1. UNIQUENESS: The description must uniquely identify this element on the current page
-2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts
-3. PRECISION: Be specific enough to distinguish from similar elements
-
-DESCRIPTION STRUCTURE:
-1. Element type (button, input, link, div, etc.)
-2. Primary identifier (in order of preference):
-   - Unique text content: "with text 'Login'"
-   - Unique attribute: "with aria-label 'Search'"
-   - Unique class/ID: "with class 'primary-button'"
-   - Unique position: "in header navigation"
-3. Secondary identifiers (if needed for uniqueness):
-   - Visual features: "blue background", "with icon"
-   - Relative position: "below search bar", "in sidebar"
-   - Parent context: "in login form", "in main menu"
-   - Neighboring stable text: "to the right of the 'Settings' section title"
+OBSERVE IN THIS ORDER:
+1. Target first: identify the smallest real UI part at the callout endpoint/center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.
+2. Primitive: name what that smallest part is before adding surrounding context.
+3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text.
+4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state.
 
-GUIDELINES:
-- Keep description under 25 words
-- Prioritize semantic identifiers over visual ones
-- Use consistent terminology across similar elements
-- Avoid page-specific or temporary content
-- Don't mention the red rectangle or selection box
-- Focus on stable, reusable characteristics
-- If the selected point/box is inside a text input, textarea, search box, or form field, describe the whole field/control, not the individual placeholder character, typed character, caret, or inner text fragment.
-- For icon-only buttons or unlabeled controls, include the nearest stable label, section title, menu item text, row text, or parent region that owns the control.
-- When multiple similar icons or controls appear in a list/sidebar/menu, the description MUST distinguish the selected one by its owning stable text or section, not by generic position such as "bottom", "nearby", or "sidebar button".
-- For expand/collapse, disclosure, chevron, close, menu, and settings icons, describe both the icon purpose and the stable text/section it controls.
-- Use the actual visible neighboring text from the current screenshot when available; do not copy labels from the examples.
+RULES:
+- Keep description under 35 words.
+- Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it.
+- Ignore every annotation overlay, including callout number, line, color, marker, border, dot, ring, crosshair, or selection box. Never describe the annotation as the target.
+- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the callout endpoint/center.
+- For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group.
+- If the endpoint/center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the endpoint/center is on that icon itself.
+- If the endpoint/center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive "dropdown" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.
+- If the endpoint/center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly under the endpoint. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.
+- If the endpoint/center is on an expanded dropdown/select/menu list item, use primitive "option" for selectable list options or "menuitem" for command menu entries.
+- Only use primitive "icon" or "arrow" when the endpoint/center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.
+- For compound controls or stacked glyphs, describe only the sub-part containing the callout endpoint/center, using upper/lower or left/right only when visible.
+- For inline text, links, or substrings, describe only the exact substring/link at the endpoint/center, not the whole line.
+- For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label.
+- For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible.
+- Use selected, highlighted, hovered, focused, or active state only if the callout endpoint/center is inside that state.
+- If the endpoint/center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.
+- Use actual visible text from the current screenshot when available; do not copy labels from the examples.
 - **Write the description in ${preferredLanguage}**
 
 EXAMPLES:
 ${getExamples(preferredLanguage)}
 
 Return JSON:
-{
-  "description": "unique element identifier",
-  "error"?: "error message if any"
-}`;
+${describeElementReturnJsonSchema()}`;
 };
 export { elementDescriberInstruction };
 

package/dist/es/ai-model/prompt/describe.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/describe.mjs","sources":["../../../../src/ai-model/prompt/describe.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\n\nconst examplesMap: Record<string, string[]> = {\n  Chinese: [\n    '\"登录表单中的\"登录\"按钮\"',\n    '\"搜索输入框，placeholder 为\"请输入关键词\"\"',\n    '\"顶部导航栏中文字为\"首页\"的链接\"',\n    '\"联系表单中的提交按钮\"',\n    '\"aria-label 为\"打开菜单\"的菜单图标\"',\n    '\"左侧导航栏中当前分组标题右侧的折叠图标\"',\n  ],\n  English: [\n    '\"Login button with text \\'Sign In\\'\"',\n    '\"Search input with placeholder \\'Enter keywords\\'\"',\n    '\"Navigation link with text \\'Home\\' in header\"',\n    '\"Submit button in contact form\"',\n    '\"Menu icon with aria-label \\'Open menu\\'\"',\n    '\"Collapse icon to the right of the current section title in the left sidebar\"',\n  ],\n};\n\nconst getExamples = (language: string) => {\n  const examples = examplesMap[language] || examplesMap.English;\n  return examples.map((e) => `- ${e}`).join('\\n');\n};\n\nexport const elementDescriberInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nDescribe the element in the red rectangle for precise identification.\n\nIMPORTANT: You MUST write the description in ${preferredLanguage}.\n\nCRITICAL REQUIREMENTS:\n1. UNIQUENESS: The description must uniquely identify this element on the current page\n2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts\n3. PRECISION: Be specific enough to distinguish from similar elements\n\nDESCRIPTION STRUCTURE:\n1. Element type (button, input, link, div, etc.)\n2. Primary identifier (in order of preference):\n   - Unique text content: \"with text 'Login'\"\n   - Unique attribute: \"with aria-label 'Search'\"\n   - Unique class/ID: \"with class 'primary-button'\"\n   - Unique position: \"in header navigation\"\n3. Secondary identifiers (if needed for uniqueness):\n   - Visual features: \"blue background\", \"with icon\"\n   - Relative position: \"below search bar\", \"in sidebar\"\n   - Parent context: \"in login form\", \"in main menu\"\n   - Neighboring stable text: \"to the right of the 'Settings' section title\"\n\nGUIDELINES:\n- Keep description under 25 words\n- Prioritize semantic identifiers over visual ones\n- Use consistent terminology across similar elements\n- Avoid page-specific or temporary content\n- Don't mention the red rectangle or selection box\n- Focus on stable, reusable characteristics\n- If the selected point/box is inside a text input, textarea, search box, or form field, describe the whole field/control, not the individual placeholder character, typed character, caret, or inner text fragment.\n- For icon-only buttons or unlabeled controls, include the nearest stable label, section title, menu item text, row text, or parent region that owns the control.\n- When multiple similar icons or controls appear in a list/sidebar/menu, the description MUST distinguish the selected one by its owning stable text or section, not by generic position such as \"bottom\", \"nearby\", or \"sidebar button\".\n- For expand/collapse, disclosure, chevron, close, menu, and settings icons, describe both the icon purpose and the stable text/section it controls.\n- Use the actual visible neighboring text from the current screenshot when available; do not copy labels from the examples.\n- **Write the description in ${preferredLanguage}**\n\nEXAMPLES:\n${getExamples(preferredLanguage)}\n\nReturn JSON:\n{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n};\n"],"names":["examplesMap","getExamples","language","examples","e","elementDescriberInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";AAEA,MAAMA,cAAwC;IAC5C,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;IACD,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;AACH;AAEA,MAAMC,cAAc,CAACC;IACnB,MAAMC,WAAWH,WAAW,CAACE,SAAS,IAAIF,YAAY,OAAO;IAC7D,OAAOG,SAAS,GAAG,CAAC,CAACC,IAAM,CAAC,EAAE,EAAEA,GAAG,EAAE,IAAI,CAAC;AAC5C;AAEO,MAAMC,8BAA8B;IACzC,MAAMC,oBAAoBC;IAE1B,OAAO,CAAC;;;6CAGmC,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;6BAgCpC,EAAEA,kBAAkB;;;AAGjD,EAAEL,YAAYK,mBAAmB;;;;;;CAMhC,CAAC;AACF"}
+{"version":3,"file":"ai-model/prompt/describe.mjs","sources":["../../../../src/ai-model/prompt/describe.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\n\nconst examplesMap: Record<string, string[]> = {\n  Chinese: [\n    '\"登录表单中的\"登录\"按钮\"',\n    '\"搜索输入框，placeholder 为\"请输入关键词\"\"',\n    '\"顶部导航栏中文字为\"首页\"的链接\"',\n    '\"联系表单中的提交按钮\"',\n    '\"aria-label 为\"打开菜单\"的菜单图标\"',\n    '\"左侧导航栏中当前分组标题右侧的折叠图标\"',\n  ],\n  English: [\n    '\"Login button with text \\'Sign In\\'\"',\n    '\"Search input with placeholder \\'Enter keywords\\'\"',\n    '\"Navigation link with text \\'Home\\' in header\"',\n    '\"Submit button in contact form\"',\n    '\"Menu icon with aria-label \\'Open menu\\'\"',\n    '\"Collapse icon to the right of the current section title in the left sidebar\"',\n  ],\n};\n\nconst getExamples = (language: string) => {\n  const examples = examplesMap[language] || examplesMap.English;\n  return examples.map((e) => `- ${e}`).join('\\n');\n};\n\nconst describeElementReturnJsonSchema = () => `{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n\nexport const elementDescriberInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nDescribe the real page element indicated by the temporary callout.\nThe callout is an annotation overlay. It is not part of the page or target.\nThe description will be used later to locate the same element on the original screenshot without annotations, so write a locator-style description instead of a visual caption.\n\nIMPORTANT: You MUST write the description in ${preferredLanguage}.\n\nOBSERVE IN THIS ORDER:\n1. Target first: identify the smallest real UI part at the callout endpoint/center: text, glyph, icon, arrow, input, dropdown/select, option, button, link, status, checkbox, radio, switch, tab, menu item, slider, image, control, or empty region.\n2. Primitive: name what that smallest part is before adding surrounding context.\n3. Owner/context: add the nearest stable owner only when it helps disambiguate, such as a label, row/card title, column header, field name, or adjacent visible text.\n4. Similar candidates: if multiple candidates look similar, add stable local anchors from the same row, card, field, header, or group. Prefer visible text and values over inferred row counting or temporary visual state.\n\nRULES:\n- Keep description under 35 words.\n- Describe the smallest indicated UI part itself, not the larger container, row, card, sentence, or group that merely contains it.\n- Ignore every annotation overlay, including callout number, line, color, marker, border, dot, ring, crosshair, or selection box. Never describe the annotation as the target.\n- Do not borrow the text, glyph, direction, purpose, or state from a nearby element outside the callout endpoint/center.\n- For tiny or icon-only controls, name the visible glyph/control and add its owner/context; adjacent text is context, not the target. If similar tiny controls are adjacent in the same group, add local order or relative position inside that group.\n- If the endpoint/center is on a field value, label, or input body, describe that value/field/control. Do not retarget to a trailing icon, dropdown arrow, clear button, or search affordance unless the endpoint/center is on that icon itself.\n- If the endpoint/center is inside the bordered body, current value, trigger, or blank area of a select/dropdown/combobox/filter field, use primitive \"dropdown\" and describe that dropdown/select control. Do not call it an input unless it is clearly a free-text field.\n- If the endpoint/center is inside the bordered body or blank area of an input or filter field, describe the field body/current value/control even when the visible text is not exactly under the endpoint. Use the field label or visible value as context; do not snap to trailing icons or nearby table headers.\n- If the endpoint/center is on an expanded dropdown/select/menu list item, use primitive \"option\" for selectable list options or \"menuitem\" for command menu entries.\n- Only use primitive \"icon\" or \"arrow\" when the endpoint/center directly overlaps the real glyph strokes. A nearby search icon, dropdown arrow, clear button, or wrong locator result must not become the target primitive.\n- For compound controls or stacked glyphs, describe only the sub-part containing the callout endpoint/center, using upper/lower or left/right only when visible.\n- For inline text, links, or substrings, describe only the exact substring/link at the endpoint/center, not the whole line.\n- For repeated rows, cards, or options, use same-local anchors that are visible in the screenshot, such as neighboring cell text, field value, title, date, time, ID, or column/header label.\n- For tables or grids, describe the target as the intersection of the target column/header and same-row anchors. Do not use row ordinals or column ordinals unless the index/header is clearly visible.\n- Use selected, highlighted, hovered, focused, or active state only if the callout endpoint/center is inside that state.\n- If the endpoint/center is on blank space, describe the empty region/gap between stable surrounding anchors. Do not invent a nearby control.\n- Use actual visible text from the current screenshot when available; do not copy labels from the examples.\n- **Write the description in ${preferredLanguage}**\n\nEXAMPLES:\n${getExamples(preferredLanguage)}\n\nReturn JSON:\n${describeElementReturnJsonSchema()}`;\n};\n"],"names":["examplesMap","getExamples","language","examples","e","describeElementReturnJsonSchema","elementDescriberInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";AAEA,MAAMA,cAAwC;IAC5C,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;IACD,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;AACH;AAEA,MAAMC,cAAc,CAACC;IACnB,MAAMC,WAAWH,WAAW,CAACE,SAAS,IAAIF,YAAY,OAAO;IAC7D,OAAOG,SAAS,GAAG,CAAC,CAACC,IAAM,CAAC,EAAE,EAAEA,GAAG,EAAE,IAAI,CAAC;AAC5C;AAEA,MAAMC,kCAAkC,IAAM,CAAC;;;CAG9C,CAAC;AAEK,MAAMC,8BAA8B;IACzC,MAAMC,oBAAoBC;IAE1B,OAAO,CAAC;;;;;6CAKmC,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;6BA0BpC,EAAEA,kBAAkB;;;AAGjD,EAAEN,YAAYM,mBAAmB;;;AAGjC,EAAEF,mCAAmC;AACrC"}

package/dist/es/ai-model/prompt/extraction.mjs

@@ -1,5 +1,5 @@
 import { getPreferredLanguage } from "@midscene/shared/env";
-import { safeParseJson } from "../service-caller/json.mjs";
+import { parseModelResponseJson } from "../service-caller/json.mjs";
 import { extractXMLTag } from "./util.mjs";
 function buildTypeQueryDemandValue(type, demand) {
     const currentScreenshotConstraint = 'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';
@@ -8,19 +8,25 @@ function buildTypeQueryDemandValue(type, demand) {
     return `${type}, ${currentScreenshotConstraint}, ${demand}`;
 }
 function parseXMLExtractionResponse(xmlString) {
-    const thought = extractXMLTag(xmlString, 'thought');
+    const thought = extractXMLTag(xmlString, 'observation');
     const dataJsonStr = extractXMLTag(xmlString, 'data-json');
     const errorsStr = extractXMLTag(xmlString, 'errors');
     if (!dataJsonStr) throw new Error('Missing required field: data-json');
     let data;
     try {
-        data = safeParseJson(dataJsonStr);
+        data = parseModelResponseJson(dataJsonStr, {
+            source: 'generic-object',
+            requireObject: false
+        });
     } catch (e) {
         throw new Error(`Failed to parse data-json: ${e}`);
     }
     let errors;
     if (errorsStr) try {
-        const parsedErrors = safeParseJson(errorsStr);
+        const parsedErrors = parseModelResponseJson(errorsStr, {
+            source: 'generic-object',
+            requireObject: false
+        });
         if (Array.isArray(parsedErrors)) errors = parsedErrors;
     } catch (e) {}
     return {
@@ -58,7 +64,7 @@ When DATA_DEMAND is a JSON object, the keys in your response must exactly match
 
 
 Return in the following XML format:
-<thought>the thinking process of the extraction, less than 300 words. Use ${preferredLanguage} in this field.</thought>
+<observation>brief evidence observed for the extraction, less than 300 words. Use ${preferredLanguage} in this field.</observation>
 <data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>
 <errors>optional error messages as JSON array, e.g., ["error1", "error2"]</errors>
 
@@ -75,7 +81,7 @@ For example, if the DATA_DEMAND is:
 
 By viewing the screenshot and page contents, you can extract the following data:
 
-<thought>According to the screenshot, i can see ...</thought>
+<observation>According to the screenshot, i can see ...</observation>
 <data-json>
 {
   "name": "John",
@@ -93,7 +99,7 @@ the todo items list, string[]
 
 By viewing the screenshot and page contents, you can extract the following data:
 
-<thought>According to the screenshot, i can see ...</thought>
+<observation>According to the screenshot, i can see ...</observation>
 <data-json>
 ["todo 1", "todo 2", "todo 3"]
 </data-json>
@@ -107,7 +113,7 @@ the page title, string
 
 By viewing the screenshot and page contents, you can extract the following data:
 
-<thought>According to the screenshot, i can see ...</thought>
+<observation>According to the screenshot, i can see ...</observation>
 <data-json>
 "todo list"
 </data-json>
@@ -123,7 +129,7 @@ If the DATA_DEMAND is:
 
 By viewing the screenshot and page contents, you can extract the following data:
 
-<thought>According to the screenshot, i can see ...</thought>
+<observation>According to the screenshot, i can see ...</observation>
 <data-json>
 { "StatementIsTruthy": true }
 </data-json>

package/dist/es/ai-model/prompt/extraction.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/extraction.mjs","sources":["../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["import type { AIDataExtractionResponse, ServiceExtractParam } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { safeParseJson } from '../service-caller/json';\nimport { extractXMLTag } from './util';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = safeParseJson(dataJsonStr) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = safeParseJson(errorsStr);\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract(options?: {\n  screenshotIncluded?: boolean;\n  referenceImagesIncluded?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n  const screenshotIncluded = options?.screenshotIncluded ?? true;\n  const referenceImagesIncluded = options?.referenceImagesIncluded ?? false;\n\n  const contextPrompts = [\n    \"The user will give you data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\",\n  ];\n\n  if (screenshotIncluded) {\n    contextPrompts.push(\n      'The user will provide a current screenshot to evaluate, and may provide its contents. Base your answer on the current screenshot and its contents when provided. Treat them as the primary source of truth for what is currently visible or true.',\n    );\n  } else {\n    contextPrompts.push(\n      'The user will not provide a current screenshot. Use only the supplied page contents and other inputs, and do not infer unsupported visual details.',\n    );\n  }\n\n  if (referenceImagesIncluded) {\n    const referenceImagesPrompt =\n      'Reference images are supporting context only unless <DATA_DEMAND> explicitly asks for comparison, matching, or reasoning about them.';\n    contextPrompts.push(\n      screenshotIncluded\n        ? `${referenceImagesPrompt} Do not conclude that something exists in the current screenshot solely because it appears in a reference image; when they conflict, trust the current screenshot and its contents.`\n        : `${referenceImagesPrompt} Do not treat reference images as direct evidence of the current state unless the demand explicitly asks you to use them that way.`,\n    );\n  }\n  const contextPrompt = contextPrompts.join('\\n\\n');\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\n${contextPrompt}\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\n\nReturn in the following XML format:\n<thought>the thinking process of the extraction, less than 300 words. Use ${preferredLanguage} in this field.</thought>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","safeParseJson","e","errors","parsedErrors","Array","systemPromptToExtract","options","preferredLanguage","getPreferredLanguage","screenshotIncluded","referenceImagesIncluded","contextPrompts","referenceImagesPrompt","contextPrompt","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;AAKO,SAASA,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAKO,SAASE,2BACdC,SAAiB;IAEjB,MAAMC,UAAUC,cAAcF,WAAW;IACzC,MAAMG,cAAcD,cAAcF,WAAW;IAC7C,MAAMI,YAAYF,cAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,cAAcJ;IACvB,EAAE,OAAOK,GAAG;QACV,MAAM,IAAIH,MAAM,CAAC,2BAA2B,EAAEG,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIL,WACF,IAAI;QACF,MAAMM,eAAeH,cAAcH;QACnC,IAAIO,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIP,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAIG,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG,sBAAsBC,OAGrC;IACC,MAAMC,oBAAoBC;IAC1B,MAAMC,qBAAqBH,SAAS,sBAAsB;IAC1D,MAAMI,0BAA0BJ,SAAS,2BAA2B;IAEpE,MAAMK,iBAAiB;QACrB;KACD;IAED,IAAIF,oBACFE,eAAe,IAAI,CACjB;SAGFA,eAAe,IAAI,CACjB;IAIJ,IAAID,yBAAyB;QAC3B,MAAME,wBACJ;QACFD,eAAe,IAAI,CACjBF,qBACI,GAAGG,sBAAsB,mLAAmL,CAAC,GAC7M,GAAGA,sBAAsB,kIAAkI,CAAC;IAEpK;IACA,MAAMC,gBAAgBF,eAAe,IAAI,CAAC;IAE1C,OAAO,CAAC;;;AAGV,EAAEE,cAAc;;;;;;;;0EAQ0D,EAAEN,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqE9F,CAAC;AACD;AAEO,MAAMO,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}
+{"version":3,"file":"ai-model/prompt/extraction.mjs","sources":["../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["import type { AIDataExtractionResponse, ServiceExtractParam } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { parseModelResponseJson } from '../service-caller/json';\nimport { extractXMLTag } from './util';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  // Keep the internal field named `thought`, but ask models to emit\n  // <observation>. Gemini may only return <thought>-named content when\n  // thinking summaries are enabled.\n  const thought = extractXMLTag(xmlString, 'observation');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = parseModelResponseJson(dataJsonStr, {\n      source: 'generic-object',\n      requireObject: false,\n    }) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = parseModelResponseJson(errorsStr, {\n        source: 'generic-object',\n        requireObject: false,\n      });\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract(options?: {\n  screenshotIncluded?: boolean;\n  referenceImagesIncluded?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n  const screenshotIncluded = options?.screenshotIncluded ?? true;\n  const referenceImagesIncluded = options?.referenceImagesIncluded ?? false;\n\n  const contextPrompts = [\n    \"The user will give you data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\",\n  ];\n\n  if (screenshotIncluded) {\n    contextPrompts.push(\n      'The user will provide a current screenshot to evaluate, and may provide its contents. Base your answer on the current screenshot and its contents when provided. Treat them as the primary source of truth for what is currently visible or true.',\n    );\n  } else {\n    contextPrompts.push(\n      'The user will not provide a current screenshot. Use only the supplied page contents and other inputs, and do not infer unsupported visual details.',\n    );\n  }\n\n  if (referenceImagesIncluded) {\n    const referenceImagesPrompt =\n      'Reference images are supporting context only unless <DATA_DEMAND> explicitly asks for comparison, matching, or reasoning about them.';\n    contextPrompts.push(\n      screenshotIncluded\n        ? `${referenceImagesPrompt} Do not conclude that something exists in the current screenshot solely because it appears in a reference image; when they conflict, trust the current screenshot and its contents.`\n        : `${referenceImagesPrompt} Do not treat reference images as direct evidence of the current state unless the demand explicitly asks you to use them that way.`,\n    );\n  }\n  const contextPrompt = contextPrompts.join('\\n\\n');\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\n${contextPrompt}\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\n\nReturn in the following XML format:\n<observation>brief evidence observed for the extraction, less than 300 words. Use ${preferredLanguage} in this field.</observation>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","parseModelResponseJson","e","errors","parsedErrors","Array","systemPromptToExtract","options","preferredLanguage","getPreferredLanguage","screenshotIncluded","referenceImagesIncluded","contextPrompts","referenceImagesPrompt","contextPrompt","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;AAKO,SAASA,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAKO,SAASE,2BACdC,SAAiB;IAKjB,MAAMC,UAAUC,cAAcF,WAAW;IACzC,MAAMG,cAAcD,cAAcF,WAAW;IAC7C,MAAMI,YAAYF,cAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,uBAAuBJ,aAAa;YACzC,QAAQ;YACR,eAAe;QACjB;IACF,EAAE,OAAOK,GAAG;QACV,MAAM,IAAIH,MAAM,CAAC,2BAA2B,EAAEG,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIL,WACF,IAAI;QACF,MAAMM,eAAeH,uBAAuBH,WAAW;YACrD,QAAQ;YACR,eAAe;QACjB;QACA,IAAIO,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIP,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAIG,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG,sBAAsBC,OAGrC;IACC,MAAMC,oBAAoBC;IAC1B,MAAMC,qBAAqBH,SAAS,sBAAsB;IAC1D,MAAMI,0BAA0BJ,SAAS,2BAA2B;IAEpE,MAAMK,iBAAiB;QACrB;KACD;IAED,IAAIF,oBACFE,eAAe,IAAI,CACjB;SAGFA,eAAe,IAAI,CACjB;IAIJ,IAAID,yBAAyB;QAC3B,MAAME,wBACJ;QACFD,eAAe,IAAI,CACjBF,qBACI,GAAGG,sBAAsB,mLAAmL,CAAC,GAC7M,GAAGA,sBAAsB,kIAAkI,CAAC;IAEpK;IACA,MAAMC,gBAAgBF,eAAe,IAAI,CAAC;IAE1C,OAAO,CAAC;;;AAGV,EAAEE,cAAc;;;;;;;;kFAQkE,EAAEN,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEtG,CAAC;AACD;AAEO,MAAMO,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}

package/dist/es/ai-model/prompt/llm-planning.mjs

@@ -112,16 +112,16 @@ async function systemPromptToTaskPlanning({ actionSpace, locatePromptSpec, inclu
     const locateExample1 = locateExample('Add to cart button for Sauce Labs Backpack', 1);
     const locateNameField = locateExample('Name input field in the registration form', 2);
     const locateEmailField = locateExample('Email input field in the registration form', 3);
-    const step1Title = shouldIncludeSubGoals ? '## Step 1: Observe and Plan (related tags: <thought>, <update-plan-content>, <mark-sub-goal-done>)' : '## Step 1: Observe (related tags: <thought>)';
+    const step1Title = shouldIncludeSubGoals ? '## Step 1: Observe and Plan (related tags: <planning>, <update-plan-content>, <mark-sub-goal-done>)' : '## Step 1: Observe (related tags: <planning>)';
     const step1Description = shouldIncludeSubGoals ? "First, observe the current screenshot and previous logs, then break down the user's instruction into multiple high-level sub-goals. Update the status of sub-goals based on what you see in the current screenshot." : 'First, observe the current screenshot and previous logs to understand the current state.';
     const explicitInstructionRule = 'CRITICAL - Following Explicit Instructions: When the user gives you specific operation steps (not high-level goals), you MUST execute ONLY those exact steps - nothing more, nothing less. Do NOT add extra actions even if they seem logical. For example: "fill out the form" means only fill fields, do NOT submit; "click the button" means only click, do NOT wait for page load or verify results; "type \'hello\'" means only type, do NOT press Enter.';
-    const thoughtTagDescription = shouldIncludeSubGoals ? `REQUIRED: You MUST always output the <thought> tag. Never skip it.
+    const planningTagDescription = shouldIncludeSubGoals ? `REQUIRED: You MUST always output the <planning> tag. Never skip it.
 
-Include your thought process in the <thought> tag. It should answer: What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action? Write your thoughts naturally without numbering or section headers.
+Include your planning details in the <planning> tag. It should answer: What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action? Write it naturally without numbering or section headers.
 
-${explicitInstructionRule}` : `REQUIRED: You MUST always output the <thought> tag. Never skip it.
+${explicitInstructionRule}` : `REQUIRED: You MUST always output the <planning> tag. Never skip it.
 
-Include your thought process in the <thought> tag. It should answer: What is the current state based on the screenshot? What should be the next action? Write your thoughts naturally without numbering or section headers.
+Include your planning details in the <planning> tag. It should answer: What is the current state based on the screenshot? What should be the next action? Write it naturally without numbering or section headers.
 
 ${explicitInstructionRule}`;
     const subGoalTags = shouldIncludeSubGoals ? `
@@ -154,7 +154,7 @@ During execution, you can call <update-plan-content> at any time to update the p
 
 If the user wants to "log in to a system using username and password, complete all to-do items, and submit a registration form", you can break it down into the following sub-goals:
 
-<thought>...</thought>
+<planning>...</planning>
 <update-plan-content>
   <sub-goal index="1" status="pending">Log in to the system</sub-goal>
   <sub-goal index="2" status="pending">Complete all to-do items</sub-goal>
@@ -190,9 +190,9 @@ ${step1Title}
 
 ${step1Description}
 ${shouldIncludeSubGoals ? `\n${OBSERVE_STEP_NOTES}\n` : ''}
-* <thought> tag (REQUIRED)
+* <planning> tag (REQUIRED)
 
-${thoughtTagDescription}
+${planningTagDescription}
 ${subGoalTags}
 ${shouldIncludeSubGoals ? `
 ## Step ${memoryStepNumber}: Memory Data from Current Screenshot (related tags: <memory>)
@@ -335,7 +335,7 @@ Return in XML format following this decision flow:
 
 **Always include (REQUIRED):**
 <!-- Step 1: Observe${shouldIncludeSubGoals ? ' and Plan' : ''} -->
-<thought>Your thought process here. NEVER skip this tag.</thought>
+<planning>Your planning details here. NEVER skip this tag.</planning>
 ${shouldIncludeSubGoals ? `
 <!-- required when no update-plan-content is provided in the previous response -->
 <update-plan-content>...</update-plan-content>
@@ -374,7 +374,7 @@ Below is an example of a multi-turn conversation for "fill out the registration
 **Screenshot:** [Shows a registration form with empty Name and Email fields]
 
 **Your response:**
-<thought>The user wants me to fill out the registration form with specific values and return the email address. I can see the form has two fields: Name and Email. Both are currently empty. I'll break this down into sub-goals and start with the Name field. Note: The instruction is to fill the form only (not submit), and return the email at the end.</thought>
+<planning>The user wants me to fill out the registration form with specific values and return the email address. I can see the form has two fields: Name and Email. Both are currently empty. I'll break this down into sub-goals and start with the Name field. Note: The instruction is to fill the form only (not submit), and return the email at the end.</planning>
 <update-plan-content>
   <sub-goal index="1" status="pending">Fill in the Name field with 'John'</sub-goal>
   <sub-goal index="2" status="pending">Fill in the Email field with 'john@example.com'</sub-goal>
@@ -404,7 +404,7 @@ Actions performed for current sub-goal:
 **Screenshot:** [Shows the form with Name field now focused/active]
 
 **Your response:**
-<thought>The Name field is now focused. I need to type 'John' into this field. Current sub-goal is running, will be completed after input.</thought>
+<planning>The Name field is now focused. I need to type 'John' into this field. Current sub-goal is running, will be completed after input.</planning>
 <log>Typing 'John' into the Name field</log>
 <action-type>Input</action-type>
 <action-param-json>
@@ -430,7 +430,7 @@ Actions performed for current sub-goal:
 **Screenshot:** [Shows the form with Name field containing 'John']
 
 **Your response:**
-<thought>The Name field now shows 'John'. Sub-goal 1 is complete. Next, I need to fill the Email field for sub-goal 2.</thought>
+<planning>The Name field now shows 'John'. Sub-goal 1 is complete. Next, I need to fill the Email field for sub-goal 2.</planning>
 <mark-sub-goal-done>
   <sub-goal index="1" status="finished" />
 </mark-sub-goal-done>
@@ -459,7 +459,7 @@ Actions performed for current sub-goal:
 **Screenshot:** [Shows the form with Name='John' and Email field focused]
 
 **Your response:**
-<thought>The Email field is now focused. I'll enter 'john@example.com'. After this, sub-goal 2 will be complete and my task will be done.</thought>
+<planning>The Email field is now focused. I'll enter 'john@example.com'. After this, sub-goal 2 will be complete and my task will be done.</planning>
 <log>Typing email address into the Email field</log>
 <action-type>Input</action-type>
 <action-param-json>
@@ -485,7 +485,7 @@ Actions performed for current sub-goal:
 **Screenshot:** [Shows the form with Name='John' and Email='john@example.com']
 
 **Your response:**
-<thought>Both fields are now filled: Name shows 'John' and Email shows 'john@example.com'. Sub-goal 2 is complete. The user asked me to return the filled email address, so I need to include 'john@example.com' in my response. All sub-goals are now finished.</thought>
+<planning>Both fields are now filled: Name shows 'John' and Email shows 'john@example.com'. Sub-goal 2 is complete. The user asked me to return the filled email address, so I need to include 'john@example.com' in my response. All sub-goals are now finished.</planning>
 <mark-sub-goal-done>
   <sub-goal index="2" status="finished" />
   <sub-goal index="3" status="finished" />
@@ -504,7 +504,7 @@ Below is an example of a multi-turn conversation for "fill out the registration
 **Screenshot:** [Shows a registration form with empty Name and Email fields]
 
 **Your response:**
-<thought>The user wants me to fill out the registration form with specific values and return the email address. I can see the form has two fields: Name and Email. Both are currently empty. I should start by clicking on the Name field. Note: The instruction is to fill the form only (not submit), and return the email at the end.</thought>
+<planning>The user wants me to fill out the registration form with specific values and return the email address. I can see the form has two fields: Name and Email. Both are currently empty. I should start by clicking on the Name field. Note: The instruction is to fill the form only (not submit), and return the email at the end.</planning>
 <log>Click on the Name field to start filling the form</log>
 <action-type>Tap</action-type>
 <action-param-json>
@@ -521,7 +521,7 @@ The previous action has been executed, here is the latest screenshot. Please con
 **Screenshot:** [Shows the form with Name field now focused/active]
 
 **Your response:**
-<thought>The Name field is now focused. I need to type 'John' into this field. After this, I'll need to fill the Email field as well.</thought>
+<planning>The Name field is now focused. I need to type 'John' into this field. After this, I'll need to fill the Email field as well.</planning>
 <log>Typing 'John' into the Name field</log>
 <action-type>Input</action-type>
 <action-param-json>
@@ -538,7 +538,7 @@ The previous action has been executed, here is the latest screenshot. Please con
 **Screenshot:** [Shows the form with Name field containing 'John']
 
 **Your response:**
-<thought>The Name field now shows 'John'. Next, I need to fill the Email field with 'john@example.com'. I'll click on the Email field first.</thought>
+<planning>The Name field now shows 'John'. Next, I need to fill the Email field with 'john@example.com'. I'll click on the Email field first.</planning>
 <log>Moving to the Email field</log>
 <action-type>Tap</action-type>
 <action-param-json>
@@ -555,7 +555,7 @@ The previous action has been executed, here is the latest screenshot. Please con
 **Screenshot:** [Shows the form with Name='John' and Email field focused]
 
 **Your response:**
-<thought>The Email field is now focused. I'll enter 'john@example.com'. After this, both required fields will be filled and my task will be complete.</thought>
+<planning>The Email field is now focused. I'll enter 'john@example.com'. After this, both required fields will be filled and my task will be complete.</planning>
 <log>Typing email address into the Email field</log>
 <action-type>Input</action-type>
 <action-param-json>
@@ -572,7 +572,7 @@ The previous action has been executed, here is the latest screenshot. Please con
 **Screenshot:** [Shows the form with Name='John' and Email='john@example.com']
 
 **Your response:**
-<thought>Both fields are now filled: Name shows 'John' and Email shows 'john@example.com'. The user asked me to return the filled email address, so I should include 'john@example.com' in my response. The instruction has been fulfilled.</thought>
+<planning>Both fields are now filled: Name shows 'John' and Email shows 'john@example.com'. The user asked me to return the filled email address, so I should include 'john@example.com' in my response. The instruction has been fulfilled.</planning>
 <complete success="true">john@example.com</complete>
 `}`;
 }