@donggui/core 1.5.4-donggui.3

package/dist/es/ai-model/prompt/llm-planning.mjs

@@ -0,0 +1,364 @@
+import { findAllMidsceneLocatorField } from "../../common.mjs";
+import { getPreferredLanguage } from "@midscene/shared/env";
+import { getZodDescription, getZodTypeName } from "@midscene/shared/zod-schema-utils";
+import { bboxDescription } from "./common.mjs";
+const vlLocateParam = (modelFamily)=>{
+    if (modelFamily) return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(modelFamily)}`;
+    return "{ prompt: string /* description of the target element */ }";
+};
+const findDefaultValue = (field)=>{
+    let current = field;
+    const visited = new Set();
+    while(current && !visited.has(current)){
+        visited.add(current);
+        const currentWithDef = current;
+        if (!currentWithDef._def?.typeName) break;
+        if ('ZodDefault' === currentWithDef._def.typeName) return currentWithDef._def.defaultValue?.();
+        if ('ZodOptional' === currentWithDef._def.typeName || 'ZodNullable' === currentWithDef._def.typeName) current = currentWithDef._def.innerType;
+        else break;
+    }
+};
+const SAMPLE_BBOXES = [
+    [
+        50,
+        100,
+        200,
+        200
+    ],
+    [
+        300,
+        400,
+        500,
+        500
+    ],
+    [
+        600,
+        100,
+        800,
+        250
+    ],
+    [
+        50,
+        600,
+        250,
+        750
+    ]
+];
+const injectBboxIntoSample = (sample, locateFields, includeBbox)=>{
+    if (!includeBbox) return sample;
+    const result = {
+        ...sample
+    };
+    let bboxIndex = 0;
+    for (const field of locateFields)if (result[field] && 'object' == typeof result[field] && result[field].prompt) {
+        result[field] = {
+            ...result[field],
+            bbox: SAMPLE_BBOXES[bboxIndex % SAMPLE_BBOXES.length]
+        };
+        bboxIndex++;
+    }
+    return result;
+};
+const descriptionForAction = (action, locatorSchemaTypeDescription, includeBbox = false)=>{
+    const tab = '  ';
+    const fields = [];
+    fields.push(`- type: "${action.name}"`);
+    if (action.paramSchema) {
+        const paramLines = [];
+        const schema = action.paramSchema;
+        const isZodObject = schema._def?.typeName === 'ZodObject';
+        if (isZodObject && schema.shape) {
+            const shape = schema.shape;
+            for (const [key, field] of Object.entries(shape))if (field && 'object' == typeof field) {
+                const isOptional = 'function' == typeof field.isOptional && field.isOptional();
+                const keyWithOptional = isOptional ? `${key}?` : key;
+                const typeName = getZodTypeName(field, locatorSchemaTypeDescription);
+                const description = getZodDescription(field);
+                const defaultValue = findDefaultValue(field);
+                const hasDefault = void 0 !== defaultValue;
+                let paramLine = `${keyWithOptional}: ${typeName}`;
+                const comments = [];
+                if (description) comments.push(description);
+                if (hasDefault) {
+                    const defaultStr = 'string' == typeof defaultValue ? `"${defaultValue}"` : JSON.stringify(defaultValue);
+                    comments.push(`default: ${defaultStr}`);
+                }
+                if (comments.length > 0) paramLine += ` // ${comments.join(', ')}`;
+                paramLines.push(paramLine);
+            }
+            if (paramLines.length > 0) {
+                fields.push('- param:');
+                paramLines.forEach((line)=>{
+                    fields.push(`  - ${line}`);
+                });
+            }
+        } else {
+            const typeName = getZodTypeName(schema);
+            const description = getZodDescription(schema);
+            let paramDescription = `- param: ${typeName}`;
+            if (description) paramDescription += ` // ${description}`;
+            paramDescription += ' (pass the value directly, not as an object)';
+            fields.push(paramDescription);
+        }
+    }
+    if (action.sample && 'object' == typeof action.sample) {
+        const locateFields = findAllMidsceneLocatorField(action.paramSchema);
+        const sampleWithBbox = injectBboxIntoSample(action.sample, locateFields, includeBbox);
+        const sampleStr = `- sample:\n${tab}${tab}<action-type>${action.name}</action-type>\n${tab}${tab}<action-param-json>\n${tab}${tab}${JSON.stringify(sampleWithBbox, null, 2).replace(/\n/g, `\n${tab}${tab}`)}\n${tab}${tab}</action-param-json>`;
+        fields.push(sampleStr);
+    }
+    return `- ${action.name}, ${action.description || "No description provided"}
+${tab}${fields.join(`\n${tab}`)}
+`.trim();
+};
+const EXPLICIT_INSTRUCTION_RULES = `
+**Explicit instructions vs. High-level goals:**
+- **Explicit steps** (e.g., "click X", "type Y", "fill form"): Execute EXACTLY those steps - nothing more
+- **High-level goals** (e.g., "log in", "complete purchase"): Determine necessary steps
+
+**Examples:**
+- "fill out the form" → Fill fields only, do NOT submit
+- "click the button" → Click only, do NOT wait for page load
+- "type 'hello'" → Type only, do NOT press Enter
+- "select the first item" → Select only, do NOT proceed to checkout`;
+const ASSERTION_RULES = `
+**Assertion instructions:**
+- If assertion condition is NOT satisfied and cannot be satisfied, mark as failed (success="false")
+- If page is loading (spinner, skeleton, progress bar), wait before asserting`;
+const PAGE_NAVIGATION_RULES = `
+**Page navigation restriction:**
+- Complete task on current page unless explicitly instructed to navigate
+- Do NOT click links, use browser navigation, or open new URLs
+- If task cannot be accomplished, report failure instead of navigating`;
+const generateSubGoalSection = ()=>`
+* <update-plan-content> tag
+
+<update-plan-content>
+  <sub-goal index="1" status="finished|pending">description</sub-goal>
+  <sub-goal index="2" status="finished|pending">description</sub-goal>
+</update-plan-content>
+
+* <mark-sub-goal-done> tag
+
+<mark-sub-goal-done>
+  <sub-goal index="1" status="finished" />
+</mark-sub-goal-done>
+
+IMPORTANT: Mark sub-goal as "finished" ONLY AFTER confirming completion in screenshot.`;
+const generateSimplifiedExample = (includeSubGoals, locateNameField, locateEmailField)=>{
+    if (includeSubGoals) return `
+## Example: Fill registration form
+
+**User:** "fill out the form with name 'John' and email 'xxxx(真实的email地址)', then return the email"
+
+**Response:**
+<thought>Form has Name and Email fields. Both empty. Break into sub-goals.</thought>
+<update-plan-content>
+  <sub-goal index="1" status="pending">Fill Name with 'John'</sub-goal>
+  <sub-goal index="2" status="pending">Fill Email with 'xxxx(真实的email地址)'</sub-goal>
+  <sub-goal index="3" status="pending">Return email address</sub-goal>
+</update-plan-content>
+<log>Click Name field</log>
+<action-type>Tap</action-type>
+<action-param-json>{"locate": ${locateNameField}}</action-param-json>
+
+(After typing name)
+<thought>Name='John'. Sub-goal 1 complete. Fill Email next.</thought>
+<mark-sub-goal-done><sub-goal index="1" status="finished" /></mark-sub-goal-done>
+<log>Click Email field</log>
+<action-type>Tap</action-type>
+<action-param-json>{"locate": ${locateEmailField}}</action-param-json>
+
+(After typing email)
+<thought>Both fields filled. All sub-goals complete.</thought>
+<mark-sub-goal-done>
+  <sub-goal index="2" status="finished" />
+  <sub-goal index="3" status="finished" />
+</mark-sub-goal-done>
+<complete success="true">xxxx(真实的email地址)</complete>`;
+    return `
+## Example: Fill registration form
+
+**User:** "fill out the form with name 'John' and email 'xxxx(真实的email地址)', then return the email"
+
+**Response:**
+<thought>Form has Name and Email fields. Both empty. Start with Name.</thought>
+<log>Click Name field</log>
+<action-type>Tap</action-type>
+<action-param-json>{"locate": ${locateNameField}}</action-param-json>
+
+(After typing name)
+<thought>Name='John'. Fill Email next.</thought>
+<log>Click Email field</log>
+<action-type>Tap</action-type>
+<action-param-json>{"locate": ${locateEmailField}}</action-param-json>
+
+(After typing email)
+<thought>Both fields filled. Task complete.</thought>
+<complete success="true">xxxx(真实的email地址)</complete>`;
+};
+async function systemPromptToTaskPlanning({ actionSpace, modelFamily, includeBbox, includeThought, includeSubGoals }) {
+    const preferredLanguage = getPreferredLanguage();
+    if (includeBbox && !modelFamily) throw new Error('modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.');
+    const actionDescriptionList = actionSpace.map((action)=>descriptionForAction(action, vlLocateParam(includeBbox ? modelFamily : void 0), includeBbox));
+    const actionList = actionDescriptionList.join('\n');
+    const shouldIncludeSubGoals = includeSubGoals ?? false;
+    const locateExample1 = includeBbox ? `{
+    "prompt": "Add to cart button for Sauce Labs Backpack",
+    "bbox": [345, 442, 458, 483]
+  }` : `{
+    "prompt": "Add to cart button for Sauce Labs Backpack"
+  }`;
+    const locateNameField = includeBbox ? `{
+    "prompt": "Name input field in the registration form",
+    "bbox": [120, 180, 380, 210]
+  }` : `{
+    "prompt": "Name input field in the registration form"
+  }`;
+    const locateEmailField = includeBbox ? `{
+    "prompt": "Email input field in the registration form",
+    "bbox": [120, 240, 380, 270]
+  }` : `{
+    "prompt": "Email input field in the registration form"
+  }`;
+    const step1Title = shouldIncludeSubGoals ? '## Step 1: Observe and Plan' : '## Step 1: Observe';
+    const step1Description = shouldIncludeSubGoals ? "Observe screenshot and logs, then break down instruction into sub-goals." : 'Observe screenshot and logs to understand current state.';
+    const thoughtTagDescription = `REQUIRED: Always output the <thought> tag.
+
+Answer these questions:
+- What is the user's requirement?
+- What is the current state?
+${shouldIncludeSubGoals ? '- Are all sub-goals completed?' : ''}
+- What should be the next action?
+
+${EXPLICIT_INSTRUCTION_RULES}`;
+    const subGoalSection = shouldIncludeSubGoals ? generateSubGoalSection() : '';
+    const memoryStepNumber = 2;
+    const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;
+    const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;
+    const goalAccomplished = shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled';
+    return `
+# Role
+You are an expert UI automation agent. Accomplish user instructions by observing screenshots and executing actions.
+
+# Steps
+
+${step1Title}
+
+${step1Description}
+
+* <thought> tag (REQUIRED)
+
+${thoughtTagDescription}
+${subGoalSection}
+${shouldIncludeSubGoals ? `
+## Step ${memoryStepNumber}: Memory Data
+
+Record information needed in follow-up actions. Current screenshot will NOT be available later.
+
+Examples: extracted data, element states, content to reference.
+` : ''}
+## Step ${checkGoalStepNumber}: Check if ${shouldIncludeSubGoals ? 'Goal is Accomplished' : 'Instruction is Fulfilled'}
+
+${shouldIncludeSubGoals ? 'Based on screenshot and sub-goals status, determine' : 'Determine'} if task is completed.
+
+### CRITICAL: User's Instruction is Supreme
+
+The user's instruction defines EXACT scope. Follow it precisely - nothing more, nothing less.
+
+${EXPLICIT_INSTRUCTION_RULES}
+
+**What "${goalAccomplished}" means:**
+- Done EXACTLY what user asked - no extra steps, no assumptions
+- Do NOT perform actions beyond explicit instruction
+
+${ASSERTION_RULES}
+${!shouldIncludeSubGoals ? `
+${PAGE_NAVIGATION_RULES}
+` : ''}
+### Output Rules
+
+- If NOT complete, continue to Step ${actionStepNumber}
+- Use <complete success="true|false">message</complete> to output result
+  - success="true" if ${goalAccomplished}
+  - success="false" if ${goalAccomplished.replace('is', 'is not').replace('fulfilled', 'fulfilled and cannot be fulfilled')}
+  - message: information for user (follow requested format)
+- If outputting <complete>, do NOT output <action-type> or <action-param-json>
+
+## Step ${actionStepNumber}: Determine Next Action
+
+ONLY if task is not complete: Determine next action based on current screenshot${shouldIncludeSubGoals ? ' and plan' : ''}.
+
+- Don't give extra actions beyond instruction${shouldIncludeSubGoals ? ' or plan' : ''}
+- Consider current screenshot - if element not visible, find it first
+- Ensure previous actions completed successfully
+- Give just the next ONE action
+- If errors persist >3 times, use <error> tag
+
+### Supporting actions list
+
+${actionList}
+
+### Log tag (preamble message)
+
+Brief message to user explaining next action:
+- **Use ${preferredLanguage}**
+- **Keep concise**: 1-2 sentences, 8-12 words
+- **Build on context**: connect with previous actions
+- **Friendly tone**: collaborative and engaging
+
+**Examples:**
+- <log>Click the login button</log>
+- <log>Scroll to find the 'Yes' button</log>
+
+### Action output
+
+Use <action-type> and <action-param-json> tags:
+- <action-type> MUST be from supporting actions list
+- Use EXACT field names from action description
+- 'complete' is NOT a valid action-type
+
+Example:
+<action-type>Tap</action-type>
+<action-param-json>
+{
+  "locate": ${locateExample1}
+}
+</action-param-json>
+
+### Error output
+
+Use <error> tag for error messages:
+<error>Unable to find the required element on the page</error>
+
+# Return Format
+
+**Always include (REQUIRED):**
+<thought>Your thought process here. NEVER skip this tag.</thought>
+${shouldIncludeSubGoals ? `
+<update-plan-content>...</update-plan-content>
+<mark-sub-goal-done>
+  <sub-goal index="1" status="finished" />
+</mark-sub-goal-done>
+` : ''}${shouldIncludeSubGoals ? `
+<memory>...</memory>
+` : ''}
+**Then choose ONE path:**
+
+**Path A: If ${goalAccomplished} or failed**
+<complete success="true|false">...</complete>
+
+**Path B: If ${goalAccomplished.replace('is', 'is not yet')}**
+<log>...</log>
+<action-type>...</action-type>
+<action-param-json>...</action-param-json>
+
+<!-- OR if there's an error -->
+<error>...</error>
+
+${generateSimplifiedExample(shouldIncludeSubGoals, locateNameField, locateEmailField)}
+`;
+}
+export { descriptionForAction, systemPromptToTaskPlanning };
+
+//# sourceMappingURL=llm-planning.mjs.map

package/dist/es/ai-model/prompt/llm-planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/llm-planning.mjs","sources":["../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/types';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport {\n  getZodDescription,\n  getZodTypeName,\n} from '@midscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { bboxDescription } from './common';\n\nconst vlLocateParam = (modelFamily: TModelFamily | undefined) => {\n  if (modelFamily) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(modelFamily)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\nconst findDefaultValue = (field: unknown): any | undefined => {\n  let current = field;\n  const visited = new Set<unknown>();\n\n  while (current && !visited.has(current)) {\n    visited.add(current);\n    const currentWithDef = current as {\n      _def?: {\n        typeName?: string;\n        defaultValue?: () => any;\n        innerType?: unknown;\n      };\n    };\n\n    if (!currentWithDef._def?.typeName) break;\n\n    if (currentWithDef._def.typeName === 'ZodDefault') {\n      return currentWithDef._def.defaultValue?.();\n    }\n\n    if (\n      currentWithDef._def.typeName === 'ZodOptional' ||\n      currentWithDef._def.typeName === 'ZodNullable'\n    ) {\n      current = currentWithDef._def.innerType;\n    } else {\n      break;\n    }\n  }\n\n  return undefined;\n};\n\nconst SAMPLE_BBOXES: [number, number, number, number][] = [\n  [50, 100, 200, 200],\n  [300, 400, 500, 500],\n  [600, 100, 800, 250],\n  [50, 600, 250, 750],\n];\n\nconst injectBboxIntoSample = (\n  sample: Record<string, any>,\n  locateFields: string[],\n  includeBbox: boolean,\n): Record<string, any> => {\n  if (!includeBbox) return sample;\n  const result = { ...sample };\n  let bboxIndex = 0;\n  for (const field of locateFields) {\n    if (\n      result[field] &&\n      typeof result[field] === 'object' &&\n      result[field].prompt\n    ) {\n      result[field] = {\n        ...result[field],\n        bbox: SAMPLE_BBOXES[bboxIndex % SAMPLE_BBOXES.length],\n      };\n      bboxIndex++;\n    }\n  }\n  return result;\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n  includeBbox = false,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  fields.push(`- type: \"${action.name}\"`);\n\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    const schema = action.paramSchema as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      const shape = schema.shape;\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          const isOptional =\n            typeof (field as { isOptional?: () => boolean }).isOptional ===\n              'function' &&\n            (field as { isOptional: () => boolean }).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          const typeName = getZodTypeName(field, locatorSchemaTypeDescription);\n\n          const description = getZodDescription(field as z.ZodTypeAny);\n\n          const defaultValue = findDefaultValue(field);\n          const hasDefault = defaultValue !== undefined;\n\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          const comments: string[] = [];\n          if (description) {\n            comments.push(description);\n          }\n          if (hasDefault) {\n            const defaultStr =\n              typeof defaultValue === 'string'\n                ? `\"${defaultValue}\"`\n                : JSON.stringify(defaultValue);\n            comments.push(`default: ${defaultStr}`);\n          }\n          if (comments.length > 0) {\n            paramLine += ` // ${comments.join(', ')}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(schema as z.ZodTypeAny);\n\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  if (action.sample && typeof action.sample === 'object') {\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n    const sampleWithBbox = injectBboxIntoSample(\n      action.sample,\n      locateFields,\n      includeBbox,\n    );\n    const sampleStr = `- sample:\\n${tab}${tab}<action-type>${action.name}</action-type>\\n${tab}${tab}<action-param-json>\\n${tab}${tab}${JSON.stringify(sampleWithBbox, null, 2).replace(/\\n/g, `\\n${tab}${tab}`)}\\n${tab}${tab}</action-param-json>`;\n    fields.push(sampleStr);\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nconst EXPLICIT_INSTRUCTION_RULES = `\n**Explicit instructions vs. High-level goals:**\n- **Explicit steps** (e.g., \"click X\", \"type Y\", \"fill form\"): Execute EXACTLY those steps - nothing more\n- **High-level goals** (e.g., \"log in\", \"complete purchase\"): Determine necessary steps\n\n**Examples:**\n- \"fill out the form\" → Fill fields only, do NOT submit\n- \"click the button\" → Click only, do NOT wait for page load\n- \"type 'hello'\" → Type only, do NOT press Enter\n- \"select the first item\" → Select only, do NOT proceed to checkout`;\n\nconst ASSERTION_RULES = `\n**Assertion instructions:**\n- If assertion condition is NOT satisfied and cannot be satisfied, mark as failed (success=\"false\")\n- If page is loading (spinner, skeleton, progress bar), wait before asserting`;\n\nconst PAGE_NAVIGATION_RULES = `\n**Page navigation restriction:**\n- Complete task on current page unless explicitly instructed to navigate\n- Do NOT click links, use browser navigation, or open new URLs\n- If task cannot be accomplished, report failure instead of navigating`;\n\nconst generateSubGoalSection = () => `\n* <update-plan-content> tag\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished|pending\">description</sub-goal>\n  <sub-goal index=\"2\" status=\"finished|pending\">description</sub-goal>\n</update-plan-content>\n\n* <mark-sub-goal-done> tag\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nIMPORTANT: Mark sub-goal as \"finished\" ONLY AFTER confirming completion in screenshot.`;\n\nconst generateSimplifiedExample = (\n  includeSubGoals: boolean,\n  locateNameField: string,\n  locateEmailField: string,\n) => {\n  if (includeSubGoals) {\n    return `\n## Example: Fill registration form\n\n**User:** \"fill out the form with name 'John' and email 'xxxx(真实的email地址)', then return the email\"\n\n**Response:**\n<thought>Form has Name and Email fields. Both empty. Break into sub-goals.</thought>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Fill Name with 'John'</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Fill Email with 'xxxx(真实的email地址)'</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Return email address</sub-goal>\n</update-plan-content>\n<log>Click Name field</log>\n<action-type>Tap</action-type>\n<action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n(After typing name)\n<thought>Name='John'. Sub-goal 1 complete. Fill Email next.</thought>\n<mark-sub-goal-done><sub-goal index=\"1\" status=\"finished\" /></mark-sub-goal-done>\n<log>Click Email field</log>\n<action-type>Tap</action-type>\n<action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n(After typing email)\n<thought>Both fields filled. All sub-goals complete.</thought>\n<mark-sub-goal-done>\n  <sub-goal index=\"2\" status=\"finished\" />\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>\n<complete success=\"true\">xxxx(真实的email地址)</complete>`;\n  }\n\n  return `\n## Example: Fill registration form\n\n**User:** \"fill out the form with name 'John' and email 'xxxx(真实的email地址)', then return the email\"\n\n**Response:**\n<thought>Form has Name and Email fields. Both empty. Start with Name.</thought>\n<log>Click Name field</log>\n<action-type>Tap</action-type>\n<action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n(After typing name)\n<thought>Name='John'. Fill Email next.</thought>\n<log>Click Email field</log>\n<action-type>Tap</action-type>\n<action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n(After typing email)\n<thought>Both fields filled. Task complete.</thought>\n<complete success=\"true\">xxxx(真实的email地址)</complete>`;\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  modelFamily,\n  includeBbox,\n  includeThought,\n  includeSubGoals,\n}: {\n  actionSpace: DeviceAction<any>[];\n  modelFamily: TModelFamily | undefined;\n  includeBbox: boolean;\n  includeThought?: boolean;\n  includeSubGoals?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n\n  if (includeBbox && !modelFamily) {\n    throw new Error(\n      'modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? modelFamily : undefined),\n      includeBbox,\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const shouldIncludeThought = includeThought ?? true;\n  const shouldIncludeSubGoals = includeSubGoals ?? false;\n\n  const locateExample1 = includeBbox\n    ? `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\",\n    \"bbox\": [345, 442, 458, 483]\n  }`\n    : `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\"\n  }`;\n\n  const locateNameField = includeBbox\n    ? `{\n    \"prompt\": \"Name input field in the registration form\",\n    \"bbox\": [120, 180, 380, 210]\n  }`\n    : `{\n    \"prompt\": \"Name input field in the registration form\"\n  }`;\n\n  const locateEmailField = includeBbox\n    ? `{\n    \"prompt\": \"Email input field in the registration form\",\n    \"bbox\": [120, 240, 380, 270]\n  }`\n    : `{\n    \"prompt\": \"Email input field in the registration form\"\n  }`;\n\n  const step1Title = shouldIncludeSubGoals\n    ? '## Step 1: Observe and Plan'\n    : '## Step 1: Observe';\n\n  const step1Description = shouldIncludeSubGoals\n    ? \"Observe screenshot and logs, then break down instruction into sub-goals.\"\n    : 'Observe screenshot and logs to understand current state.';\n\n  const thoughtTagDescription = `REQUIRED: Always output the <thought> tag.\n\nAnswer these questions:\n- What is the user's requirement?\n- What is the current state?\n${shouldIncludeSubGoals ? '- Are all sub-goals completed?' : ''}\n- What should be the next action?\n\n${EXPLICIT_INSTRUCTION_RULES}`;\n\n  const subGoalSection = shouldIncludeSubGoals ? generateSubGoalSection() : '';\n\n  const memoryStepNumber = 2;\n  const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;\n  const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;\n\n  const goalTerm = shouldIncludeSubGoals ? 'goal' : 'instruction';\n  const goalAccomplished = shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled';\n\n  return `\n# Role\nYou are an expert UI automation agent. Accomplish user instructions by observing screenshots and executing actions.\n\n# Steps\n\n${step1Title}\n\n${step1Description}\n\n* <thought> tag (REQUIRED)\n\n${thoughtTagDescription}\n${subGoalSection}\n${\n  shouldIncludeSubGoals\n    ? `\n## Step ${memoryStepNumber}: Memory Data\n\nRecord information needed in follow-up actions. Current screenshot will NOT be available later.\n\nExamples: extracted data, element states, content to reference.\n`\n    : ''\n}\n## Step ${checkGoalStepNumber}: Check if ${shouldIncludeSubGoals ? 'Goal is Accomplished' : 'Instruction is Fulfilled'}\n\n${shouldIncludeSubGoals ? 'Based on screenshot and sub-goals status, determine' : 'Determine'} if task is completed.\n\n### CRITICAL: User's Instruction is Supreme\n\nThe user's instruction defines EXACT scope. Follow it precisely - nothing more, nothing less.\n\n${EXPLICIT_INSTRUCTION_RULES}\n\n**What \"${goalAccomplished}\" means:**\n- Done EXACTLY what user asked - no extra steps, no assumptions\n- Do NOT perform actions beyond explicit instruction\n\n${ASSERTION_RULES}\n${\n  !shouldIncludeSubGoals\n    ? `\n${PAGE_NAVIGATION_RULES}\n`\n    : ''\n}\n### Output Rules\n\n- If NOT complete, continue to Step ${actionStepNumber}\n- Use <complete success=\"true|false\">message</complete> to output result\n  - success=\"true\" if ${goalAccomplished}\n  - success=\"false\" if ${goalAccomplished.replace('is', 'is not').replace('fulfilled', 'fulfilled and cannot be fulfilled')}\n  - message: information for user (follow requested format)\n- If outputting <complete>, do NOT output <action-type> or <action-param-json>\n\n## Step ${actionStepNumber}: Determine Next Action\n\nONLY if task is not complete: Determine next action based on current screenshot${shouldIncludeSubGoals ? ' and plan' : ''}.\n\n- Don't give extra actions beyond instruction${shouldIncludeSubGoals ? ' or plan' : ''}\n- Consider current screenshot - if element not visible, find it first\n- Ensure previous actions completed successfully\n- Give just the next ONE action\n- If errors persist >3 times, use <error> tag\n\n### Supporting actions list\n\n${actionList}\n\n### Log tag (preamble message)\n\nBrief message to user explaining next action:\n- **Use ${preferredLanguage}**\n- **Keep concise**: 1-2 sentences, 8-12 words\n- **Build on context**: connect with previous actions\n- **Friendly tone**: collaborative and engaging\n\n**Examples:**\n- <log>Click the login button</log>\n- <log>Scroll to find the 'Yes' button</log>\n\n### Action output\n\nUse <action-type> and <action-param-json> tags:\n- <action-type> MUST be from supporting actions list\n- Use EXACT field names from action description\n- 'complete' is NOT a valid action-type\n\nExample:\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateExample1}\n}\n</action-param-json>\n\n### Error output\n\nUse <error> tag for error messages:\n<error>Unable to find the required element on the page</error>\n\n# Return Format\n\n**Always include (REQUIRED):**\n<thought>Your thought process here. NEVER skip this tag.</thought>\n${\n  shouldIncludeSubGoals\n    ? `\n<update-plan-content>...</update-plan-content>\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n`\n    : ''\n}${\n  shouldIncludeSubGoals\n    ? `\n<memory>...</memory>\n`\n    : ''\n}\n**Then choose ONE path:**\n\n**Path A: If ${goalAccomplished} or failed**\n<complete success=\"true|false\">...</complete>\n\n**Path B: If ${goalAccomplished.replace('is', 'is not yet')}**\n<log>...</log>\n<action-type>...</action-type>\n<action-param-json>...</action-param-json>\n\n<!-- OR if there's an error -->\n<error>...</error>\n\n${generateSimplifiedExample(shouldIncludeSubGoals, locateNameField, locateEmailField)}\n`;\n}\n"],"names":["vlLocateParam","modelFamily","bboxDescription","findDefaultValue","field","current","visited","Set","currentWithDef","SAMPLE_BBOXES","injectBboxIntoSample","sample","locateFields","includeBbox","result","bboxIndex","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","key","Object","isOptional","keyWithOptional","typeName","getZodTypeName","description","getZodDescription","defaultValue","hasDefault","undefined","paramLine","comments","defaultStr","JSON","line","paramDescription","findAllMidsceneLocatorField","sampleWithBbox","sampleStr","EXPLICIT_INSTRUCTION_RULES","ASSERTION_RULES","PAGE_NAVIGATION_RULES","generateSubGoalSection","generateSimplifiedExample","includeSubGoals","locateNameField","locateEmailField","systemPromptToTaskPlanning","actionSpace","includeThought","preferredLanguage","getPreferredLanguage","Error","actionDescriptionList","actionList","shouldIncludeSubGoals","locateExample1","step1Title","step1Description","thoughtTagDescription","subGoalSection","memoryStepNumber","checkGoalStepNumber","actionStepNumber","goalAccomplished"],"mappings":";;;;AAWA,MAAMA,gBAAgB,CAACC;IACrB,IAAIA,aACF,OAAO,CAAC,6DAA6D,EAAEC,gBAAgBD,cAAc;IAEvG,OAAO;AACT;AAEA,MAAME,mBAAmB,CAACC;IACxB,IAAIC,UAAUD;IACd,MAAME,UAAU,IAAIC;IAEpB,MAAOF,WAAW,CAACC,QAAQ,GAAG,CAACD,SAAU;QACvCC,QAAQ,GAAG,CAACD;QACZ,MAAMG,iBAAiBH;QAQvB,IAAI,CAACG,eAAe,IAAI,EAAE,UAAU;QAEpC,IAAIA,AAAiC,iBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAC9B,OAAOA,eAAe,IAAI,CAAC,YAAY;QAGzC,IACEA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,IAC5BA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAE5BH,UAAUG,eAAe,IAAI,CAAC,SAAS;aAEvC;IAEJ;AAGF;AAEA,MAAMC,gBAAoD;IACxD;QAAC;QAAI;QAAK;QAAK;KAAI;IACnB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAI;QAAK;QAAK;KAAI;CACpB;AAED,MAAMC,uBAAuB,CAC3BC,QACAC,cACAC;IAEA,IAAI,CAACA,aAAa,OAAOF;IACzB,MAAMG,SAAS;QAAE,GAAGH,MAAM;IAAC;IAC3B,IAAII,YAAY;IAChB,KAAK,MAAMX,SAASQ,aAClB,IACEE,MAAM,CAACV,MAAM,IACb,AAAyB,YAAzB,OAAOU,MAAM,CAACV,MAAM,IACpBU,MAAM,CAACV,MAAM,CAAC,MAAM,EACpB;QACAU,MAAM,CAACV,MAAM,GAAG;YACd,GAAGU,MAAM,CAACV,MAAM;YAChB,MAAMK,aAAa,CAACM,YAAYN,cAAc,MAAM,CAAC;QACvD;QACAM;IACF;IAEF,OAAOD;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC,8BACAL,cAAc,KAAK;IAEnB,MAAMM,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAE3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAEtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAE/B,MAAMC,SAASL,OAAO,WAAW;QAIjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAC/B,MAAME,QAAQF,OAAO,KAAK;YAE1B,KAAK,MAAM,CAACG,KAAKrB,MAAM,IAAIsB,OAAO,OAAO,CAACF,OACxC,IAAIpB,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBACtC,MAAMuB,aACJ,AACE,cADF,OAAQvB,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBACrD,MAAMwB,kBAAkBD,aAAa,GAAGF,IAAI,CAAC,CAAC,GAAGA;gBAEjD,MAAMI,WAAWC,eAAe1B,OAAOc;gBAEvC,MAAMa,cAAcC,kBAAkB5B;gBAEtC,MAAM6B,eAAe9B,iBAAiBC;gBACtC,MAAM8B,aAAaD,AAAiBE,WAAjBF;gBAEnB,IAAIG,YAAY,GAAGR,gBAAgB,EAAE,EAAEC,UAAU;gBACjD,MAAMQ,WAAqB,EAAE;gBAC7B,IAAIN,aACFM,SAAS,IAAI,CAACN;gBAEhB,IAAIG,YAAY;oBACd,MAAMI,aACJ,AAAwB,YAAxB,OAAOL,eACH,CAAC,CAAC,EAAEA,aAAa,CAAC,CAAC,GACnBM,KAAK,SAAS,CAACN;oBACrBI,SAAS,IAAI,CAAC,CAAC,SAAS,EAAEC,YAAY;gBACxC;gBACA,IAAID,SAAS,MAAM,GAAG,GACpBD,aAAa,CAAC,IAAI,EAAEC,SAAS,IAAI,CAAC,OAAO;gBAG3ChB,WAAW,IAAI,CAACe;YAClB;YAGF,IAAIf,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACmB;oBAClBpB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEoB,MAAM;gBAC3B;YACF;QACF,OAAO;YACL,MAAMX,WAAWC,eAAeR;YAChC,MAAMS,cAAcC,kBAAkBV;YAEtC,IAAImB,mBAAmB,CAAC,SAAS,EAAEZ,UAAU;YAC7C,IAAIE,aACFU,oBAAoB,CAAC,IAAI,EAAEV,aAAa;YAE1CU,oBAAoB;YAEpBrB,OAAO,IAAI,CAACqB;QACd;IACF;IAEA,IAAIxB,OAAO,MAAM,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EAAe;QACtD,MAAML,eAAe8B,4BAA4BzB,OAAO,WAAW;QACnE,MAAM0B,iBAAiBjC,qBACrBO,OAAO,MAAM,EACbL,cACAC;QAEF,MAAM+B,YAAY,CAAC,WAAW,EAAEzB,MAAMA,IAAI,aAAa,EAAEF,OAAO,IAAI,CAAC,gBAAgB,EAAEE,MAAMA,IAAI,qBAAqB,EAAEA,MAAMA,MAAMoB,KAAK,SAAS,CAACI,gBAAgB,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,EAAExB,MAAMA,KAAK,EAAE,EAAE,EAAEA,MAAMA,IAAI,oBAAoB,CAAC;QAChPC,OAAO,IAAI,CAACwB;IACd;IAEA,OAAO,CAAC,EAAE,EAAE3B,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEA,MAAM0B,6BAA6B,CAAC;;;;;;;;;mEAS+B,CAAC;AAEpE,MAAMC,kBAAkB,CAAC;;;6EAGoD,CAAC;AAE9E,MAAMC,wBAAwB,CAAC;;;;sEAIuC,CAAC;AAEvE,MAAMC,yBAAyB,IAAM,CAAC;;;;;;;;;;;;;;sFAcgD,CAAC;AAEvF,MAAMC,4BAA4B,CAChCC,iBACAC,iBACAC;IAEA,IAAIF,iBACF,OAAO,CAAC;;;;;;;;;;;;;;8BAckB,EAAEC,gBAAgB;;;;;;;8BAOlB,EAAEC,iBAAiB;;;;;;;;oDAQG,CAAC;IAGnD,OAAO,CAAC;;;;;;;;;8BASoB,EAAED,gBAAgB;;;;;;8BAMlB,EAAEC,iBAAiB;;;;oDAIG,CAAC;AACrD;AAEO,eAAeC,2BAA2B,EAC/CC,WAAW,EACXrD,WAAW,EACXY,WAAW,EACX0C,cAAc,EACdL,eAAe,EAOhB;IACC,MAAMM,oBAAoBC;IAE1B,IAAI5C,eAAe,CAACZ,aAClB,MAAM,IAAIyD,MACR;IAIJ,MAAMC,wBAAwBL,YAAY,GAAG,CAAC,CAACrC,SACtCD,qBACLC,QACAjB,cAAca,cAAcZ,cAAckC,SAC1CtB;IAGJ,MAAM+C,aAAaD,sBAAsB,IAAI,CAAC;IAG9C,MAAME,wBAAwBX,mBAAmB;IAEjD,MAAMY,iBAAiBjD,cACnB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAMsC,kBAAkBtC,cACpB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAMuC,mBAAmBvC,cACrB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAMkD,aAAaF,wBACf,gCACA;IAEJ,MAAMG,mBAAmBH,wBACrB,6EACA;IAEJ,MAAMI,wBAAwB,CAAC;;;;;AAKjC,EAAEJ,wBAAwB,mCAAmC,GAAG;;;AAGhE,EAAEhB,4BAA4B;IAE5B,MAAMqB,iBAAiBL,wBAAwBb,2BAA2B;IAE1E,MAAMmB,mBAAmB;IACzB,MAAMC,sBAAsBP,wBAAwB,IAAI;IACxD,MAAMQ,mBAAmBR,wBAAwB,IAAI;IAGrD,MAAMS,mBAAmBT,wBAAwB,yBAAyB;IAE1E,OAAO,CAAC;;;;;;AAMV,EAAEE,WAAW;;AAEb,EAAEC,iBAAiB;;;;AAInB,EAAEC,sBAAsB;AACxB,EAAEC,eAAe;AACjB,EACEL,wBACI,CAAC;QACC,EAAEM,iBAAiB;;;;;AAK3B,CAAC,GACK,GACL;QACO,EAAEC,oBAAoB,WAAW,EAAEP,wBAAwB,yBAAyB,2BAA2B;;AAEvH,EAAEA,wBAAwB,wDAAwD,YAAY;;;;;;AAM9F,EAAEhB,2BAA2B;;QAErB,EAAEyB,iBAAiB;;;;AAI3B,EAAExB,gBAAgB;AAClB,EACE,CAACe,wBACG,CAAC;AACP,EAAEd,sBAAsB;AACxB,CAAC,GACK,GACL;;;oCAGmC,EAAEsB,iBAAiB;;sBAEjC,EAAEC,iBAAiB;uBAClB,EAAEA,iBAAiB,OAAO,CAAC,MAAM,UAAU,OAAO,CAAC,aAAa,qCAAqC;;;;QAIpH,EAAED,iBAAiB;;+EAEoD,EAAER,wBAAwB,cAAc,GAAG;;6CAE7E,EAAEA,wBAAwB,aAAa,GAAG;;;;;;;;AAQvF,EAAED,WAAW;;;;;QAKL,EAAEJ,kBAAkB;;;;;;;;;;;;;;;;;;;;YAoBhB,EAAEM,eAAe;;;;;;;;;;;;;AAa7B,EACED,wBACI,CAAC;;;;;AAKP,CAAC,GACK,KAEJA,wBACI,CAAC;;AAEP,CAAC,GACK,GACL;;;aAGY,EAAES,iBAAiB;;;aAGnB,EAAEA,iBAAiB,OAAO,CAAC,MAAM,cAAc;;;;;;;;AAQ5D,EAAErB,0BAA0BY,uBAAuBV,iBAAiBC,kBAAkB;AACtF,CAAC;AACD"}

package/dist/es/ai-model/prompt/llm-section-locator.mjs

@@ -0,0 +1,44 @@
+import { getPreferredLanguage } from "@midscene/shared/env";
+import { bboxDescription } from "./common.mjs";
+function systemPromptToLocateSection(modelFamily) {
+    const preferredLanguage = getPreferredLanguage();
+    const bboxFormat = bboxDescription(modelFamily);
+    return `
+## Role:
+You are an AI assistant that helps identify UI elements.
+
+## Objective:
+- Find a section containing the target element
+- If the description mentions reference elements, also locate sections containing those references
+
+## Output Format:
+\`\`\`json
+{
+  "bbox": [number, number, number, number],  // ${bboxFormat}
+  "references_bbox"?: [
+    [number, number, number, number],
+    ...
+  ],
+  "error"?: string
+}
+\`\`\`
+
+Fields:
+* \`bbox\` - Bounding box of the section containing the target element
+* \`references_bbox\` - Optional array of bounding boxes for reference elements
+* \`error\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.
+
+Example:
+If the description is "delete button on the second row with title 'Peter'", return:
+\`\`\`json
+{
+  "bbox": [100, 100, 200, 200],
+  "references_bbox": [[100, 100, 200, 200]]
+}
+\`\`\`
+`;
+}
+const sectionLocatorInstruction = (sectionDescription)=>`Find section containing: ${sectionDescription}`;
+export { sectionLocatorInstruction, systemPromptToLocateSection };
+
+//# sourceMappingURL=llm-section-locator.mjs.map

package/dist/es/ai-model/prompt/llm-section-locator.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/llm-section-locator.mjs","sources":["../../../../src/ai-model/prompt/llm-section-locator.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { bboxDescription } from './common';\n\nexport function systemPromptToLocateSection(\n  modelFamily: TModelFamily | undefined,\n) {\n  const preferredLanguage = getPreferredLanguage();\n  const bboxFormat = bboxDescription(modelFamily);\n  return `\n## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Find a section containing the target element\n- If the description mentions reference elements, also locate sections containing those references\n\n## Output Format:\n\\`\\`\\`json\n{\n  \"bbox\": [number, number, number, number],  // ${bboxFormat}\n  \"references_bbox\"?: [\n    [number, number, number, number],\n    ...\n  ],\n  \"error\"?: string\n}\n\\`\\`\\`\n\nFields:\n* \\`bbox\\` - Bounding box of the section containing the target element\n* \\`references_bbox\\` - Optional array of bounding boxes for reference elements\n* \\`error\\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.\n\nExample:\nIf the description is \"delete button on the second row with title 'Peter'\", return:\n\\`\\`\\`json\n{\n  \"bbox\": [100, 100, 200, 200],\n  \"references_bbox\": [[100, 100, 200, 200]]\n}\n\\`\\`\\`\n`;\n}\n\nexport const sectionLocatorInstruction = (sectionDescription: string) =>\n  `Find section containing: ${sectionDescription}`;\n"],"names":["systemPromptToLocateSection","modelFamily","preferredLanguage","getPreferredLanguage","bboxFormat","bboxDescription","sectionLocatorInstruction","sectionDescription"],"mappings":";;AAIO,SAASA,4BACdC,WAAqC;IAErC,MAAMC,oBAAoBC;IAC1B,MAAMC,aAAaC,gBAAgBJ;IACnC,OAAO,CAAC;;;;;;;;;;;gDAWsC,EAAEG,WAAW;;;;;;;;;;;;yEAYY,EAAEF,kBAAkB;;;;;;;;;;AAU7F,CAAC;AACD;AAEO,MAAMI,4BAA4B,CAACC,qBACxC,CAAC,yBAAyB,EAAEA,oBAAoB"}

package/dist/es/ai-model/prompt/order-sensitive-judge.mjs

@@ -0,0 +1,35 @@
+function systemPromptToJudgeOrderSensitive() {
+    return `
+## Role:
+You are an AI assistant that analyzes UI element descriptions.
+
+## Objective:
+Determine whether a given element description is order-sensitive.
+
+Order-sensitive descriptions contain phrases that specify position or sequence, such as:
+- "the first button"
+- "the second item"
+- "the third row"
+- "the last input"
+- "the 5th element"
+
+Order-insensitive descriptions do not specify position:
+- "login button"
+- "search input"
+- "submit button"
+- "user avatar"
+
+## Output Format:
+\`\`\`json
+{
+  "isOrderSensitive": boolean
+}
+\`\`\`
+
+Return true if the description is order-sensitive, false otherwise.
+`;
+}
+const orderSensitiveJudgePrompt = (description)=>`Analyze this element description: "${description}"`;
+export { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive };
+
+//# sourceMappingURL=order-sensitive-judge.mjs.map

package/dist/es/ai-model/prompt/order-sensitive-judge.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/order-sensitive-judge.mjs","sources":["../../../../src/ai-model/prompt/order-sensitive-judge.ts"],"sourcesContent":["export function systemPromptToJudgeOrderSensitive() {\n  return `\n## Role:\nYou are an AI assistant that analyzes UI element descriptions.\n\n## Objective:\nDetermine whether a given element description is order-sensitive.\n\nOrder-sensitive descriptions contain phrases that specify position or sequence, such as:\n- \"the first button\"\n- \"the second item\"\n- \"the third row\"\n- \"the last input\"\n- \"the 5th element\"\n\nOrder-insensitive descriptions do not specify position:\n- \"login button\"\n- \"search input\"\n- \"submit button\"\n- \"user avatar\"\n\n## Output Format:\n\\`\\`\\`json\n{\n  \"isOrderSensitive\": boolean\n}\n\\`\\`\\`\n\nReturn true if the description is order-sensitive, false otherwise.\n`;\n}\n\nexport const orderSensitiveJudgePrompt = (description: string) => {\n  return `Analyze this element description: \"${description}\"`;\n};\n"],"names":["systemPromptToJudgeOrderSensitive","orderSensitiveJudgePrompt","description"],"mappings":"AAAO,SAASA;IACd,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4BV,CAAC;AACD;AAEO,MAAMC,4BAA4B,CAACC,cACjC,CAAC,mCAAmC,EAAEA,YAAY,CAAC,CAAC"}

package/dist/es/ai-model/prompt/playwright-generator.mjs

@@ -0,0 +1,117 @@
+import { PLAYWRIGHT_EXAMPLE_CODE } from "@midscene/shared/constants";
+import { callAI, callAIWithStringResponse } from "../index.mjs";
+import { createEventCounts, createMessageContent, extractInputDescriptions, filterEventsByType, getScreenshotsForLLM, prepareEventSummary, processEventsForLLM, validateEvents } from "./yaml-generator.mjs";
+const generatePlaywrightTest = async (events, options, modelConfig)=>{
+    validateEvents(events);
+    const summary = prepareEventSummary(events, {
+        testName: options.testName,
+        maxScreenshots: options.maxScreenshots || 3
+    });
+    const playwrightSummary = {
+        ...summary,
+        waitForNetworkIdle: false !== options.waitForNetworkIdle,
+        waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,
+        viewportSize: options.viewportSize || {
+            width: 1280,
+            height: 800
+        }
+    };
+    const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);
+    const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.
+
+Event Summary:
+${JSON.stringify(playwrightSummary, null, 2)}
+
+Generated code should:
+1. Import required dependencies
+2. Set up the test with proper configuration
+3. Include a beforeEach hook to navigate to the starting URL
+4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)
+5. Include appropriate assertions and validations
+6. Follow best practices for Playwright tests
+7. Be ready to execute without further modification
+
+Important: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \`\`\`typescript, \`\`\`javascript or \`\`\`). Start directly with the code content.`;
+    const messageContent = createMessageContent(promptText, screenshots, false !== options.includeScreenshots);
+    const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. 
+Your task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.
+
+${PLAYWRIGHT_EXAMPLE_CODE}`;
+    const prompt = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: messageContent
+        }
+    ];
+    const response = await callAIWithStringResponse(prompt, modelConfig);
+    if (response?.content && 'string' == typeof response.content) return response.content;
+    throw new Error('Failed to generate Playwright test code');
+};
+const generatePlaywrightTestStream = async (events, options, modelConfig)=>{
+    validateEvents(events);
+    const summary = prepareEventSummary(events, {
+        testName: options.testName,
+        maxScreenshots: options.maxScreenshots || 3
+    });
+    const playwrightSummary = {
+        ...summary,
+        waitForNetworkIdle: false !== options.waitForNetworkIdle,
+        waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,
+        viewportSize: options.viewportSize || {
+            width: 1280,
+            height: 800
+        }
+    };
+    const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);
+    const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.
+
+Event Summary:
+${JSON.stringify(playwrightSummary, null, 2)}
+
+Generated code should:
+1. Import required dependencies
+2. Set up the test with proper configuration
+3. Include a beforeEach hook to navigate to the starting URL
+4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)
+5. Include appropriate assertions and validations
+6. Follow best practices for Playwright tests
+7. Be ready to execute without further modification
+8. can't wrap this test code in markdown code block
+
+Important: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \`\`\`typescript, \`\`\`javascript or \`\`\`). Start directly with the code content.`;
+    const messageContent = createMessageContent(promptText, screenshots, false !== options.includeScreenshots);
+    const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. 
+Your task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.
+
+${PLAYWRIGHT_EXAMPLE_CODE}`;
+    const prompt = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: messageContent
+        }
+    ];
+    if (options.stream && options.onChunk) return await callAI(prompt, modelConfig, {
+        stream: true,
+        onChunk: options.onChunk
+    });
+    {
+        const response = await callAIWithStringResponse(prompt, modelConfig);
+        if (response?.content && 'string' == typeof response.content) return {
+            content: response.content,
+            usage: response.usage,
+            isStreamed: false
+        };
+        throw new Error('Failed to generate Playwright test code');
+    }
+};
+export { createEventCounts, createMessageContent, extractInputDescriptions, filterEventsByType, generatePlaywrightTest, generatePlaywrightTestStream, getScreenshotsForLLM, prepareEventSummary, processEventsForLLM, validateEvents };
+
+//# sourceMappingURL=playwright-generator.mjs.map

package/dist/es/ai-model/prompt/playwright-generator.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/playwright-generator.mjs","sources":["../../../../src/ai-model/prompt/playwright-generator.ts"],"sourcesContent":["import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { callAI, callAIWithStringResponse } from '../index';\n// Import shared utilities and types from yaml-generator\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './yaml-generator';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(prompt, modelConfig);\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAI(prompt, modelConfig, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(prompt, modelConfig);\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["generatePlaywrightTest","events","options","modelConfig","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","Error","generatePlaywrightTestStream","callAI"],"mappings":";;;AA6DO,MAAMA,yBAAyB,OACpCC,QACAC,SACAC;IAGAC,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;8LAWiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,yBAAyBF,QAAQb;IAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIE,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CnB,QACAC,SACAC;IAGAC,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;8LAYiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMmB,OAAOL,QAAQb,aAAa;QACvC,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,yBAAyBF,QAAQb;QAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB;AACF"}