@midscene/core 1.0.1-beta-20251205094204.0 → 1.0.1-beta-20251208031856.0

package/dist/lib/ai-model/prompt/llm-planning.js

@@ -24,8 +24,9 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
+    systemPromptToTaskPlanning: ()=>systemPromptToTaskPlanning,
     descriptionForAction: ()=>descriptionForAction,
-    systemPromptToTaskPlanning: ()=>systemPromptToTaskPlanning
+    planSchema: ()=>planSchema
 });
 const external_common_js_namespaceObject = require("../../common.js");
 const external_common_js_namespaceObject_1 = require("./common.js");
@@ -163,7 +164,7 @@ Please tell what the next one action is (or null if no action should be done) to
 - Make sure the previous actions are completed successfully before performing the next step
 - If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 5 times, you should think this is an error and set the "error" field to the error message.
 - If there is nothing to do but waiting, set the "sleep" field to the positive waiting time in milliseconds and null for the "action" field.
-- Assertions are also important steps. When getting the assertion instruction, a solid conclusion is required. You should explicitly state your conclusion in the "log" field like this: "Assert: <condition>. I can see <...>, and I think <...>, so the assertion is <truthy / falsy>". If the assertion is falsy, think this an fatal error and set the reason into the "error" field.
+- When the next step is to assert something, this is a very important step, you should think about it carefully and give a solid result. Write your result in the "log" field like this: "Assert: <condition>. I think <...>, so the result is <true / false>". You don't need to give the next one action when you are asserting something. If the assertion result is false, think this an fatal error and set the reason into the "error" field. If the assertion result is true, you can continue to the next step.
 
 ## Supporting actions
 ${actionList}
@@ -178,36 +179,113 @@ Return in JSON format:
   ${commonOutputFields}
   "action": 
     {
-      "type": string, // the type of the action
-      "param"?: { // The parameter of the action, if any
-         // k-v style parameter fields
-      }, 
+      // one of the supporting actions
     } | null,
   ,
   "sleep"?: number, // The sleep time after the action, in milliseconds.
 }
-
-For example, if the instruction is to login and the form has already been filled, this is a valid return value:
-
-{
-  "log": "Click the login button",
-  "more_actions_needed_by_instruction": false,
-  "action": {
-    "type": "Tap",
-    "param": {
-      "locate": { 
-        "prompt": "The login button"${vlMode ? ', "bbox": [100, 200, 300, 400]' : ''}
-      }
-    }
-  }
 `;
 }
+const planSchema = {
+    type: 'json_schema',
+    json_schema: {
+        name: 'action_items',
+        strict: false,
+        schema: {
+            type: 'object',
+            strict: false,
+            properties: {
+                actions: {
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        strict: false,
+                        properties: {
+                            thought: {
+                                type: 'string',
+                                description: 'Reasons for generating this task, and why this task is feasible on this page'
+                            },
+                            type: {
+                                type: 'string',
+                                description: 'Type of action'
+                            },
+                            param: {
+                                anyOf: [
+                                    {
+                                        type: 'null'
+                                    },
+                                    {
+                                        type: 'object',
+                                        additionalProperties: true
+                                    }
+                                ],
+                                description: 'Parameter of the action'
+                            },
+                            locate: {
+                                type: [
+                                    'object',
+                                    'null'
+                                ],
+                                properties: {
+                                    id: {
+                                        type: 'string'
+                                    },
+                                    prompt: {
+                                        type: 'string'
+                                    }
+                                },
+                                required: [
+                                    'id',
+                                    'prompt'
+                                ],
+                                additionalProperties: false,
+                                description: 'Location information for the target element'
+                            }
+                        },
+                        required: [
+                            'thought',
+                            'type',
+                            'param',
+                            'locate'
+                        ],
+                        additionalProperties: false
+                    },
+                    description: 'List of actions to be performed'
+                },
+                more_actions_needed_by_instruction: {
+                    type: 'boolean',
+                    description: 'If all the actions described in the instruction have been covered by this action and logs, set this field to false.'
+                },
+                log: {
+                    type: 'string',
+                    description: 'Log what these planned actions do. Do not include further actions that have not been planned.'
+                },
+                error: {
+                    type: [
+                        'string',
+                        'null'
+                    ],
+                    description: 'Error messages about unexpected situations'
+                }
+            },
+            required: [
+                'actions',
+                'more_actions_needed_by_instruction',
+                'log',
+                'error'
+            ],
+            additionalProperties: false
+        }
+    }
+};
 exports.descriptionForAction = __webpack_exports__.descriptionForAction;
+exports.planSchema = __webpack_exports__.planSchema;
 exports.systemPromptToTaskPlanning = __webpack_exports__.systemPromptToTaskPlanning;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "descriptionForAction",
+    "planSchema",
     "systemPromptToTaskPlanning"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/llm-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/types';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport type { z } from 'zod';\nimport { ifMidsceneLocatorField } from '../../common';\nimport { bboxDescription } from './common';\n\n// Note: put the log field first to trigger the CoT\n\nconst commonOutputFields = `\"error\"?: string, // Error messages about unexpected situations, if any. Only think it is an error when the situation is not foreseeable according to the instruction. Use the same language as the user's instruction.\n  \"more_actions_needed_by_instruction\": boolean, // Consider if there is still more action(s) to do after the action in \"Log\" is done, according to the instruction. If so, set this field to true. Otherwise, set it to false.`;\n\nconst vlLocateParam = (vlMode: TVlModeTypes | undefined) => {\n  if (vlMode) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(vlMode)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  // Add the action type field\n  fields.push(`- type: \"${action.name}\"`);\n\n  // Handle paramSchema if it exists\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    // Check if paramSchema is a ZodObject with shape\n    const schema = action.paramSchema as any;\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      // Original logic for ZodObject schemas\n      const shape = schema.shape;\n\n      // Helper function to get type name from zod schema\n      const getTypeName = (field: any): string => {\n        // Recursively unwrap optional, nullable, and other wrapper types to get the actual inner type\n        const unwrapField = (f: any): any => {\n          if (!f._def) return f;\n\n          const typeName = f._def.typeName;\n\n          // Handle wrapper types that have innerType\n          if (\n            typeName === 'ZodOptional' ||\n            typeName === 'ZodNullable' ||\n            typeName === 'ZodDefault'\n          ) {\n            return unwrapField(f._def.innerType);\n          }\n\n          // Handle ZodEffects (transformations, refinements, preprocessors)\n          if (typeName === 'ZodEffects') {\n            // For ZodEffects, unwrap the schema field which contains the underlying type\n            if (f._def.schema) {\n              return unwrapField(f._def.schema);\n            }\n          }\n\n          return f;\n        };\n\n        const actualField = unwrapField(field);\n        const fieldTypeName = actualField._def?.typeName;\n\n        if (fieldTypeName === 'ZodString') return 'string';\n        if (fieldTypeName === 'ZodNumber') return 'number';\n        if (fieldTypeName === 'ZodBoolean') return 'boolean';\n        if (fieldTypeName === 'ZodArray') return 'array';\n        if (fieldTypeName === 'ZodObject') {\n          // Check if this is a passthrough object (like MidsceneLocation)\n          if (ifMidsceneLocatorField(actualField)) {\n            return locatorSchemaTypeDescription;\n          }\n          return 'object';\n        }\n        if (fieldTypeName === 'ZodEnum') {\n          const values =\n            (actualField._def?.values as unknown[] | undefined)\n              ?.map((option: unknown) => String(`'${option}'`))\n              .join(', ') ?? 'enum';\n\n          return `enum(${values})`;\n        }\n        // Handle ZodUnion by taking the first option (for display purposes)\n        if (fieldTypeName === 'ZodUnion') {\n          const options = actualField._def?.options as any[] | undefined;\n          if (options && options.length > 0) {\n            // For unions, list all types\n            const types = options.map((opt: any) => getTypeName(opt));\n            return types.join(' | ');\n          }\n          return 'union';\n        }\n\n        console.warn(\n          'failed to parse Zod type. This may lead to wrong params from the LLM.\\n',\n          actualField._def,\n        );\n        return actualField.toString();\n      };\n\n      // Helper function to get description from zod schema\n      const getDescription = (field: z.ZodTypeAny): string | null => {\n        // Recursively unwrap optional, nullable, and other wrapper types to get the actual inner type\n        const unwrapField = (f: any): any => {\n          if (!f._def) return f;\n\n          const typeName = f._def.typeName;\n\n          // Handle wrapper types that have innerType\n          if (\n            typeName === 'ZodOptional' ||\n            typeName === 'ZodNullable' ||\n            typeName === 'ZodDefault'\n          ) {\n            return unwrapField(f._def.innerType);\n          }\n\n          // Handle ZodEffects (transformations, refinements, preprocessors)\n          if (typeName === 'ZodEffects') {\n            // For ZodEffects, unwrap the schema field which contains the underlying type\n            if (f._def.schema) {\n              return unwrapField(f._def.schema);\n            }\n          }\n\n          return f;\n        };\n\n        // Check for direct description on the original field (wrapper may have description)\n        if ('description' in field) {\n          return field.description || null;\n        }\n\n        const actualField = unwrapField(field);\n\n        // Check for description on the unwrapped field\n        if ('description' in actualField) {\n          return actualField.description || null;\n        }\n\n        // Check for MidsceneLocation fields and add description\n        if (actualField._def?.typeName === 'ZodObject') {\n          if ('midscene_location_field_flag' in actualField._def.shape()) {\n            return 'Location information for the target element';\n          }\n        }\n\n        return null;\n      };\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          // Check if field is optional\n          const isOptional =\n            typeof (field as any).isOptional === 'function' &&\n            (field as any).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          // Get the type name\n          const typeName = getTypeName(field);\n\n          // Get description\n          const description = getDescription(field as z.ZodTypeAny);\n\n          // Build param line for this field\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          if (description) {\n            paramLine += ` // ${description}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      // Add the param section to fields if there are paramLines\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      // Handle non-object schemas (string, number, etc.)\n      // For simple primitive types, the param should be passed directly as the value\n      const schemaTypeName = schema._def?.typeName;\n      let typeName = 'unknown';\n\n      if (schemaTypeName === 'ZodString') typeName = 'string';\n      else if (schemaTypeName === 'ZodNumber') typeName = 'number';\n      else if (schemaTypeName === 'ZodBoolean') typeName = 'boolean';\n\n      // Get description if available\n      const description = 'description' in schema ? schema.description : null;\n\n      // For simple types, indicate that param should be the direct value, not an object\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  vlMode,\n  includeBbox,\n}: {\n  actionSpace: DeviceAction<any>[];\n  vlMode: TVlModeTypes | undefined;\n  includeBbox: boolean;\n}) {\n  // Validate parameters: if includeBbox is true, vlMode must be defined\n  if (includeBbox && !vlMode) {\n    throw new Error(\n      'vlMode cannot be undefined when includeBbox is true. A valid vlMode is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? vlMode : undefined),\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const logFieldInstruction = `\n## About the \\`log\\` field (preamble message)\n\nThe \\`log\\` field is a brief preamble message to the user explaining what you’re about to do. It should follow these principles and examples:\n\n- **Use the same language as the user's instruction**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what’s been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- \"Click the login button\"\n- \"Scroll to find the 'Yes' button in popup\"\n- \"Previous actions failed to find the 'Yes' button, i will try again\"\n- \"Go back to find the login button\"\n`;\n\n  return `\nTarget: User will give you an instruction, some screenshots and previous logs indicating what have been done. Your task is to plan the next one action according to current situation to accomplish the instruction.\n\nPlease tell what the next one action is (or null if no action should be done) to do the tasks the instruction requires. \n\n## Rules\n\n- Don't give extra actions or plans beyond the instruction. For example, don't try to submit the form if the instruction is only to fill something.\n- Give just the next ONE action you should do\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully before performing the next step\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 5 times, you should think this is an error and set the \"error\" field to the error message.\n- If there is nothing to do but waiting, set the \"sleep\" field to the positive waiting time in milliseconds and null for the \"action\" field.\n- Assertions are also important steps. When getting the assertion instruction, a solid conclusion is required. You should explicitly state your conclusion in the \"log\" field like this: \"Assert: <condition>. I can see <...>, and I think <...>, so the assertion is <truthy / falsy>\". If the assertion is falsy, think this an fatal error and set the reason into the \"error\" field.\n\n## Supporting actions\n${actionList}\n\n${logFieldInstruction}\n\n## Return format\n\nReturn in JSON format:\n{\n  \"log\": string, // a brief preamble to the user explaining what you’re about to do\n  ${commonOutputFields}\n  \"action\": \n    {\n      \"type\": string, // the type of the action\n      \"param\"?: { // The parameter of the action, if any\n         // k-v style parameter fields\n      }, \n    } | null,\n  ,\n  \"sleep\"?: number, // The sleep time after the action, in milliseconds.\n}\n\nFor example, if the instruction is to login and the form has already been filled, this is a valid return value:\n\n{\n  \"log\": \"Click the login button\",\n  \"more_actions_needed_by_instruction\": false,\n  \"action\": {\n    \"type\": \"Tap\",\n    \"param\": {\n      \"locate\": { \n        \"prompt\": \"The login button\"${vlMode ? `, \"bbox\": [100, 200, 300, 400]` : ''}\n      }\n    }\n  }\n`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","commonOutputFields","vlLocateParam","vlMode","bboxDescription","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","getTypeName","field","unwrapField","f","typeName","actualField","fieldTypeName","ifMidsceneLocatorField","values","option","String","options","types","opt","console","getDescription","isOptional","keyWithOptional","description","paramLine","line","schemaTypeName","paramDescription","systemPromptToTaskPlanning","actionSpace","includeBbox","Error","actionDescriptionList","undefined","actionList","logFieldInstruction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACEA,MAAMI,qBAAqB,CAAC;+NACmM,CAAC;AAEhO,MAAMC,gBAAgB,CAACC;IACrB,IAAIA,QACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EAAgBD,SAAS;IAElG,OAAO;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC;IAEA,MAAMC,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QACjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAG1B,MAAMG,cAAc,CAACC;gBAEnB,MAAMC,cAAc,CAACC;oBACnB,IAAI,CAACA,EAAE,IAAI,EAAE,OAAOA;oBAEpB,MAAMC,WAAWD,EAAE,IAAI,CAAC,QAAQ;oBAGhC,IACEC,AAAa,kBAAbA,YACAA,AAAa,kBAAbA,YACAA,AAAa,iBAAbA,UAEA,OAAOF,YAAYC,EAAE,IAAI,CAAC,SAAS;oBAIrC,IAAIC,AAAa,iBAAbA,UAEF;wBAAA,IAAID,EAAE,IAAI,CAAC,MAAM,EACf,OAAOD,YAAYC,EAAE,IAAI,CAAC,MAAM;oBAClC;oBAGF,OAAOA;gBACT;gBAEA,MAAME,cAAcH,YAAYD;gBAChC,MAAMK,gBAAgBD,YAAY,IAAI,EAAE;gBAExC,IAAIC,AAAkB,gBAAlBA,eAA+B,OAAO;gBAC1C,IAAIA,AAAkB,gBAAlBA,eAA+B,OAAO;gBAC1C,IAAIA,AAAkB,iBAAlBA,eAAgC,OAAO;gBAC3C,IAAIA,AAAkB,eAAlBA,eAA8B,OAAO;gBACzC,IAAIA,AAAkB,gBAAlBA,eAA+B;oBAEjC,IAAIC,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBF,cACzB,OAAOZ;oBAET,OAAO;gBACT;gBACA,IAAIa,AAAkB,cAAlBA,eAA6B;oBAC/B,MAAME,SACHH,YAAY,IAAI,EAAE,QACf,IAAI,CAACI,SAAoBC,OAAO,CAAC,CAAC,EAAED,OAAO,CAAC,CAAC,GAC9C,KAAK,SAAS;oBAEnB,OAAO,CAAC,KAAK,EAAED,OAAO,CAAC,CAAC;gBAC1B;gBAEA,IAAIF,AAAkB,eAAlBA,eAA8B;oBAChC,MAAMK,UAAUN,YAAY,IAAI,EAAE;oBAClC,IAAIM,WAAWA,QAAQ,MAAM,GAAG,GAAG;wBAEjC,MAAMC,QAAQD,QAAQ,GAAG,CAAC,CAACE,MAAab,YAAYa;wBACpD,OAAOD,MAAM,IAAI,CAAC;oBACpB;oBACA,OAAO;gBACT;gBAEAE,QAAQ,IAAI,CACV,2EACAT,YAAY,IAAI;gBAElB,OAAOA,YAAY,QAAQ;YAC7B;YAGA,MAAMU,iBAAiB,CAACd;gBAEtB,MAAMC,cAAc,CAACC;oBACnB,IAAI,CAACA,EAAE,IAAI,EAAE,OAAOA;oBAEpB,MAAMC,WAAWD,EAAE,IAAI,CAAC,QAAQ;oBAGhC,IACEC,AAAa,kBAAbA,YACAA,AAAa,kBAAbA,YACAA,AAAa,iBAAbA,UAEA,OAAOF,YAAYC,EAAE,IAAI,CAAC,SAAS;oBAIrC,IAAIC,AAAa,iBAAbA,UAEF;wBAAA,IAAID,EAAE,IAAI,CAAC,MAAM,EACf,OAAOD,YAAYC,EAAE,IAAI,CAAC,MAAM;oBAClC;oBAGF,OAAOA;gBACT;gBAGA,IAAI,iBAAiBF,OACnB,OAAOA,MAAM,WAAW,IAAI;gBAG9B,MAAMI,cAAcH,YAAYD;gBAGhC,IAAI,iBAAiBI,aACnB,OAAOA,YAAY,WAAW,IAAI;gBAIpC,IAAIA,YAAY,IAAI,EAAE,aAAa,aACjC;oBAAA,IAAI,kCAAkCA,YAAY,IAAI,CAAC,KAAK,IAC1D,OAAO;gBACT;gBAGF,OAAO;YACT;YAEA,KAAK,MAAM,CAACvB,KAAKmB,MAAM,IAAIlB,OAAO,OAAO,CAACgB,OACxC,IAAIE,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMe,aACJ,AAAqC,cAArC,OAAQf,MAAc,UAAU,IAC/BA,MAAc,UAAU;gBAC3B,MAAMgB,kBAAkBD,aAAa,GAAGlC,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMsB,WAAWJ,YAAYC;gBAG7B,MAAMiB,cAAcH,eAAed;gBAGnC,IAAIkB,YAAY,GAAGF,gBAAgB,EAAE,EAAEb,UAAU;gBACjD,IAAIc,aACFC,aAAa,CAAC,IAAI,EAAED,aAAa;gBAGnCtB,WAAW,IAAI,CAACuB;YAClB;YAIF,IAAIvB,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACwB;oBAClBzB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEyB,MAAM;gBAC3B;YACF;QACF,OAAO;YAGL,MAAMC,iBAAiBxB,OAAO,IAAI,EAAE;YACpC,IAAIO,WAAW;YAEf,IAAIiB,AAAmB,gBAAnBA,gBAAgCjB,WAAW;iBAC1C,IAAIiB,AAAmB,gBAAnBA,gBAAgCjB,WAAW;iBAC/C,IAAIiB,AAAmB,iBAAnBA,gBAAiCjB,WAAW;YAGrD,MAAMc,cAAc,iBAAiBrB,SAASA,OAAO,WAAW,GAAG;YAGnE,IAAIyB,mBAAmB,CAAC,SAAS,EAAElB,UAAU;YAC7C,IAAIc,aACFI,oBAAoB,CAAC,IAAI,EAAEJ,aAAa;YAE1CI,oBAAoB;YAEpB3B,OAAO,IAAI,CAAC2B;QACd;IACF;IAEA,OAAO,CAAC,EAAE,EAAE9B,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAe6B,2BAA2B,EAC/CC,WAAW,EACXnC,MAAM,EACNoC,WAAW,EAKZ;IAEC,IAAIA,eAAe,CAACpC,QAClB,MAAM,IAAIqC,MACR;IAIJ,MAAMC,wBAAwBH,YAAY,GAAG,CAAC,CAAChC,SACtCD,qBACLC,QACAJ,cAAcqC,cAAcpC,SAASuC;IAGzC,MAAMC,aAAaF,sBAAsB,IAAI,CAAC;IAE9C,MAAMG,sBAAsB,CAAC;;;;;;;;;;;;;;;AAe/B,CAAC;IAEC,OAAO,CAAC;;;;;;;;;;;;;;;;AAgBV,EAAED,WAAW;;AAEb,EAAEC,oBAAoB;;;;;;;EAOpB,EAAE3C,mBAAmB;;;;;;;;;;;;;;;;;;;;;oCAqBa,EAAEE,SAAS,mCAAmC,GAAG;;;;AAIrF,CAAC;AACD"}
+{"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/types';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\nimport type { z } from 'zod';\nimport { ifMidsceneLocatorField } from '../../common';\nimport { bboxDescription } from './common';\n\n// Note: put the log field first to trigger the CoT\n\nconst commonOutputFields = `\"error\"?: string, // Error messages about unexpected situations, if any. Only think it is an error when the situation is not foreseeable according to the instruction. Use the same language as the user's instruction.\n  \"more_actions_needed_by_instruction\": boolean, // Consider if there is still more action(s) to do after the action in \"Log\" is done, according to the instruction. If so, set this field to true. Otherwise, set it to false.`;\n\nconst vlLocateParam = (vlMode: TVlModeTypes | undefined) => {\n  if (vlMode) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(vlMode)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  // Add the action type field\n  fields.push(`- type: \"${action.name}\"`);\n\n  // Handle paramSchema if it exists\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    // Check if paramSchema is a ZodObject with shape\n    const schema = action.paramSchema as any;\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      // Original logic for ZodObject schemas\n      const shape = schema.shape;\n\n      // Helper function to get type name from zod schema\n      const getTypeName = (field: any): string => {\n        // Recursively unwrap optional, nullable, and other wrapper types to get the actual inner type\n        const unwrapField = (f: any): any => {\n          if (!f._def) return f;\n\n          const typeName = f._def.typeName;\n\n          // Handle wrapper types that have innerType\n          if (\n            typeName === 'ZodOptional' ||\n            typeName === 'ZodNullable' ||\n            typeName === 'ZodDefault'\n          ) {\n            return unwrapField(f._def.innerType);\n          }\n\n          // Handle ZodEffects (transformations, refinements, preprocessors)\n          if (typeName === 'ZodEffects') {\n            // For ZodEffects, unwrap the schema field which contains the underlying type\n            if (f._def.schema) {\n              return unwrapField(f._def.schema);\n            }\n          }\n\n          return f;\n        };\n\n        const actualField = unwrapField(field);\n        const fieldTypeName = actualField._def?.typeName;\n\n        if (fieldTypeName === 'ZodString') return 'string';\n        if (fieldTypeName === 'ZodNumber') return 'number';\n        if (fieldTypeName === 'ZodBoolean') return 'boolean';\n        if (fieldTypeName === 'ZodArray') return 'array';\n        if (fieldTypeName === 'ZodObject') {\n          // Check if this is a passthrough object (like MidsceneLocation)\n          if (ifMidsceneLocatorField(actualField)) {\n            return locatorSchemaTypeDescription;\n          }\n          return 'object';\n        }\n        if (fieldTypeName === 'ZodEnum') {\n          const values =\n            (actualField._def?.values as unknown[] | undefined)\n              ?.map((option: unknown) => String(`'${option}'`))\n              .join(', ') ?? 'enum';\n\n          return `enum(${values})`;\n        }\n        // Handle ZodUnion by taking the first option (for display purposes)\n        if (fieldTypeName === 'ZodUnion') {\n          const options = actualField._def?.options as any[] | undefined;\n          if (options && options.length > 0) {\n            // For unions, list all types\n            const types = options.map((opt: any) => getTypeName(opt));\n            return types.join(' | ');\n          }\n          return 'union';\n        }\n\n        console.warn(\n          'failed to parse Zod type. This may lead to wrong params from the LLM.\\n',\n          actualField._def,\n        );\n        return actualField.toString();\n      };\n\n      // Helper function to get description from zod schema\n      const getDescription = (field: z.ZodTypeAny): string | null => {\n        // Recursively unwrap optional, nullable, and other wrapper types to get the actual inner type\n        const unwrapField = (f: any): any => {\n          if (!f._def) return f;\n\n          const typeName = f._def.typeName;\n\n          // Handle wrapper types that have innerType\n          if (\n            typeName === 'ZodOptional' ||\n            typeName === 'ZodNullable' ||\n            typeName === 'ZodDefault'\n          ) {\n            return unwrapField(f._def.innerType);\n          }\n\n          // Handle ZodEffects (transformations, refinements, preprocessors)\n          if (typeName === 'ZodEffects') {\n            // For ZodEffects, unwrap the schema field which contains the underlying type\n            if (f._def.schema) {\n              return unwrapField(f._def.schema);\n            }\n          }\n\n          return f;\n        };\n\n        // Check for direct description on the original field (wrapper may have description)\n        if ('description' in field) {\n          return field.description || null;\n        }\n\n        const actualField = unwrapField(field);\n\n        // Check for description on the unwrapped field\n        if ('description' in actualField) {\n          return actualField.description || null;\n        }\n\n        // Check for MidsceneLocation fields and add description\n        if (actualField._def?.typeName === 'ZodObject') {\n          if ('midscene_location_field_flag' in actualField._def.shape()) {\n            return 'Location information for the target element';\n          }\n        }\n\n        return null;\n      };\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          // Check if field is optional\n          const isOptional =\n            typeof (field as any).isOptional === 'function' &&\n            (field as any).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          // Get the type name\n          const typeName = getTypeName(field);\n\n          // Get description\n          const description = getDescription(field as z.ZodTypeAny);\n\n          // Build param line for this field\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          if (description) {\n            paramLine += ` // ${description}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      // Add the param section to fields if there are paramLines\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      // Handle non-object schemas (string, number, etc.)\n      // For simple primitive types, the param should be passed directly as the value\n      const schemaTypeName = schema._def?.typeName;\n      let typeName = 'unknown';\n\n      if (schemaTypeName === 'ZodString') typeName = 'string';\n      else if (schemaTypeName === 'ZodNumber') typeName = 'number';\n      else if (schemaTypeName === 'ZodBoolean') typeName = 'boolean';\n\n      // Get description if available\n      const description = 'description' in schema ? schema.description : null;\n\n      // For simple types, indicate that param should be the direct value, not an object\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  vlMode,\n  includeBbox,\n}: {\n  actionSpace: DeviceAction<any>[];\n  vlMode: TVlModeTypes | undefined;\n  includeBbox: boolean;\n}) {\n  // Validate parameters: if includeBbox is true, vlMode must be defined\n  if (includeBbox && !vlMode) {\n    throw new Error(\n      'vlMode cannot be undefined when includeBbox is true. A valid vlMode is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? vlMode : undefined),\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const logFieldInstruction = `\n## About the \\`log\\` field (preamble message)\n\nThe \\`log\\` field is a brief preamble message to the user explaining what you’re about to do. It should follow these principles and examples:\n\n- **Use the same language as the user's instruction**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what’s been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- \"Click the login button\"\n- \"Scroll to find the 'Yes' button in popup\"\n- \"Previous actions failed to find the 'Yes' button, i will try again\"\n- \"Go back to find the login button\"\n`;\n\n  return `\nTarget: User will give you an instruction, some screenshots and previous logs indicating what have been done. Your task is to plan the next one action according to current situation to accomplish the instruction.\n\nPlease tell what the next one action is (or null if no action should be done) to do the tasks the instruction requires. \n\n## Rules\n\n- Don't give extra actions or plans beyond the instruction. For example, don't try to submit the form if the instruction is only to fill something.\n- Give just the next ONE action you should do\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully before performing the next step\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 5 times, you should think this is an error and set the \"error\" field to the error message.\n- If there is nothing to do but waiting, set the \"sleep\" field to the positive waiting time in milliseconds and null for the \"action\" field.\n- When the next step is to assert something, this is a very important step, you should think about it carefully and give a solid result. Write your result in the \"log\" field like this: \"Assert: <condition>. I think <...>, so the result is <true / false>\". You don't need to give the next one action when you are asserting something. If the assertion result is false, think this an fatal error and set the reason into the \"error\" field. If the assertion result is true, you can continue to the next step.\n\n## Supporting actions\n${actionList}\n\n${logFieldInstruction}\n\n## Return format\n\nReturn in JSON format:\n{\n  \"log\": string, // a brief preamble to the user explaining what you’re about to do\n  ${commonOutputFields}\n  \"action\": \n    {\n      // one of the supporting actions\n    } | null,\n  ,\n  \"sleep\"?: number, // The sleep time after the action, in milliseconds.\n}\n`;\n}\n\nexport const planSchema: ResponseFormatJSONSchema = {\n  type: 'json_schema',\n  json_schema: {\n    name: 'action_items',\n    strict: false,\n    schema: {\n      type: 'object',\n      strict: false,\n      properties: {\n        actions: {\n          type: 'array',\n          items: {\n            type: 'object',\n            strict: false,\n            properties: {\n              thought: {\n                type: 'string',\n                description:\n                  'Reasons for generating this task, and why this task is feasible on this page',\n              },\n              type: {\n                type: 'string',\n                description: 'Type of action',\n              },\n              param: {\n                anyOf: [\n                  { type: 'null' },\n                  {\n                    type: 'object',\n                    additionalProperties: true,\n                  },\n                ],\n                description: 'Parameter of the action',\n              },\n              locate: {\n                type: ['object', 'null'],\n                properties: {\n                  id: { type: 'string' },\n                  prompt: { type: 'string' },\n                },\n                required: ['id', 'prompt'],\n                additionalProperties: false,\n                description: 'Location information for the target element',\n              },\n            },\n            required: ['thought', 'type', 'param', 'locate'],\n            additionalProperties: false,\n          },\n          description: 'List of actions to be performed',\n        },\n        more_actions_needed_by_instruction: {\n          type: 'boolean',\n          description:\n            'If all the actions described in the instruction have been covered by this action and logs, set this field to false.',\n        },\n        log: {\n          type: 'string',\n          description:\n            'Log what these planned actions do. Do not include further actions that have not been planned.',\n        },\n        error: {\n          type: ['string', 'null'],\n          description: 'Error messages about unexpected situations',\n        },\n      },\n      required: [\n        'actions',\n        'more_actions_needed_by_instruction',\n        'log',\n        'error',\n      ],\n      additionalProperties: false,\n    },\n  },\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","commonOutputFields","vlLocateParam","vlMode","bboxDescription","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","getTypeName","field","unwrapField","f","typeName","actualField","fieldTypeName","ifMidsceneLocatorField","values","option","String","options","types","opt","console","getDescription","isOptional","keyWithOptional","description","paramLine","line","schemaTypeName","paramDescription","systemPromptToTaskPlanning","actionSpace","includeBbox","Error","actionDescriptionList","undefined","actionList","logFieldInstruction","planSchema"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACGA,MAAMI,qBAAqB,CAAC;+NACmM,CAAC;AAEhO,MAAMC,gBAAgB,CAACC;IACrB,IAAIA,QACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EAAgBD,SAAS;IAElG,OAAO;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC;IAEA,MAAMC,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QACjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAG1B,MAAMG,cAAc,CAACC;gBAEnB,MAAMC,cAAc,CAACC;oBACnB,IAAI,CAACA,EAAE,IAAI,EAAE,OAAOA;oBAEpB,MAAMC,WAAWD,EAAE,IAAI,CAAC,QAAQ;oBAGhC,IACEC,AAAa,kBAAbA,YACAA,AAAa,kBAAbA,YACAA,AAAa,iBAAbA,UAEA,OAAOF,YAAYC,EAAE,IAAI,CAAC,SAAS;oBAIrC,IAAIC,AAAa,iBAAbA,UAEF;wBAAA,IAAID,EAAE,IAAI,CAAC,MAAM,EACf,OAAOD,YAAYC,EAAE,IAAI,CAAC,MAAM;oBAClC;oBAGF,OAAOA;gBACT;gBAEA,MAAME,cAAcH,YAAYD;gBAChC,MAAMK,gBAAgBD,YAAY,IAAI,EAAE;gBAExC,IAAIC,AAAkB,gBAAlBA,eAA+B,OAAO;gBAC1C,IAAIA,AAAkB,gBAAlBA,eAA+B,OAAO;gBAC1C,IAAIA,AAAkB,iBAAlBA,eAAgC,OAAO;gBAC3C,IAAIA,AAAkB,eAAlBA,eAA8B,OAAO;gBACzC,IAAIA,AAAkB,gBAAlBA,eAA+B;oBAEjC,IAAIC,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBF,cACzB,OAAOZ;oBAET,OAAO;gBACT;gBACA,IAAIa,AAAkB,cAAlBA,eAA6B;oBAC/B,MAAME,SACHH,YAAY,IAAI,EAAE,QACf,IAAI,CAACI,SAAoBC,OAAO,CAAC,CAAC,EAAED,OAAO,CAAC,CAAC,GAC9C,KAAK,SAAS;oBAEnB,OAAO,CAAC,KAAK,EAAED,OAAO,CAAC,CAAC;gBAC1B;gBAEA,IAAIF,AAAkB,eAAlBA,eAA8B;oBAChC,MAAMK,UAAUN,YAAY,IAAI,EAAE;oBAClC,IAAIM,WAAWA,QAAQ,MAAM,GAAG,GAAG;wBAEjC,MAAMC,QAAQD,QAAQ,GAAG,CAAC,CAACE,MAAab,YAAYa;wBACpD,OAAOD,MAAM,IAAI,CAAC;oBACpB;oBACA,OAAO;gBACT;gBAEAE,QAAQ,IAAI,CACV,2EACAT,YAAY,IAAI;gBAElB,OAAOA,YAAY,QAAQ;YAC7B;YAGA,MAAMU,iBAAiB,CAACd;gBAEtB,MAAMC,cAAc,CAACC;oBACnB,IAAI,CAACA,EAAE,IAAI,EAAE,OAAOA;oBAEpB,MAAMC,WAAWD,EAAE,IAAI,CAAC,QAAQ;oBAGhC,IACEC,AAAa,kBAAbA,YACAA,AAAa,kBAAbA,YACAA,AAAa,iBAAbA,UAEA,OAAOF,YAAYC,EAAE,IAAI,CAAC,SAAS;oBAIrC,IAAIC,AAAa,iBAAbA,UAEF;wBAAA,IAAID,EAAE,IAAI,CAAC,MAAM,EACf,OAAOD,YAAYC,EAAE,IAAI,CAAC,MAAM;oBAClC;oBAGF,OAAOA;gBACT;gBAGA,IAAI,iBAAiBF,OACnB,OAAOA,MAAM,WAAW,IAAI;gBAG9B,MAAMI,cAAcH,YAAYD;gBAGhC,IAAI,iBAAiBI,aACnB,OAAOA,YAAY,WAAW,IAAI;gBAIpC,IAAIA,YAAY,IAAI,EAAE,aAAa,aACjC;oBAAA,IAAI,kCAAkCA,YAAY,IAAI,CAAC,KAAK,IAC1D,OAAO;gBACT;gBAGF,OAAO;YACT;YAEA,KAAK,MAAM,CAACvB,KAAKmB,MAAM,IAAIlB,OAAO,OAAO,CAACgB,OACxC,IAAIE,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMe,aACJ,AAAqC,cAArC,OAAQf,MAAc,UAAU,IAC/BA,MAAc,UAAU;gBAC3B,MAAMgB,kBAAkBD,aAAa,GAAGlC,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMsB,WAAWJ,YAAYC;gBAG7B,MAAMiB,cAAcH,eAAed;gBAGnC,IAAIkB,YAAY,GAAGF,gBAAgB,EAAE,EAAEb,UAAU;gBACjD,IAAIc,aACFC,aAAa,CAAC,IAAI,EAAED,aAAa;gBAGnCtB,WAAW,IAAI,CAACuB;YAClB;YAIF,IAAIvB,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACwB;oBAClBzB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEyB,MAAM;gBAC3B;YACF;QACF,OAAO;YAGL,MAAMC,iBAAiBxB,OAAO,IAAI,EAAE;YACpC,IAAIO,WAAW;YAEf,IAAIiB,AAAmB,gBAAnBA,gBAAgCjB,WAAW;iBAC1C,IAAIiB,AAAmB,gBAAnBA,gBAAgCjB,WAAW;iBAC/C,IAAIiB,AAAmB,iBAAnBA,gBAAiCjB,WAAW;YAGrD,MAAMc,cAAc,iBAAiBrB,SAASA,OAAO,WAAW,GAAG;YAGnE,IAAIyB,mBAAmB,CAAC,SAAS,EAAElB,UAAU;YAC7C,IAAIc,aACFI,oBAAoB,CAAC,IAAI,EAAEJ,aAAa;YAE1CI,oBAAoB;YAEpB3B,OAAO,IAAI,CAAC2B;QACd;IACF;IAEA,OAAO,CAAC,EAAE,EAAE9B,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAe6B,2BAA2B,EAC/CC,WAAW,EACXnC,MAAM,EACNoC,WAAW,EAKZ;IAEC,IAAIA,eAAe,CAACpC,QAClB,MAAM,IAAIqC,MACR;IAIJ,MAAMC,wBAAwBH,YAAY,GAAG,CAAC,CAAChC,SACtCD,qBACLC,QACAJ,cAAcqC,cAAcpC,SAASuC;IAGzC,MAAMC,aAAaF,sBAAsB,IAAI,CAAC;IAE9C,MAAMG,sBAAsB,CAAC;;;;;;;;;;;;;;;AAe/B,CAAC;IAEC,OAAO,CAAC;;;;;;;;;;;;;;;;AAgBV,EAAED,WAAW;;AAEb,EAAEC,oBAAoB;;;;;;;EAOpB,EAAE3C,mBAAmB;;;;;;;;AAQvB,CAAC;AACD;AAEO,MAAM4C,aAAuC;IAClD,MAAM;IACN,aAAa;QACX,MAAM;QACN,QAAQ;QACR,QAAQ;YACN,MAAM;YACN,QAAQ;YACR,YAAY;gBACV,SAAS;oBACP,MAAM;oBACN,OAAO;wBACL,MAAM;wBACN,QAAQ;wBACR,YAAY;4BACV,SAAS;gCACP,MAAM;gCACN,aACE;4BACJ;4BACA,MAAM;gCACJ,MAAM;gCACN,aAAa;4BACf;4BACA,OAAO;gCACL,OAAO;oCACL;wCAAE,MAAM;oCAAO;oCACf;wCACE,MAAM;wCACN,sBAAsB;oCACxB;iCACD;gCACD,aAAa;4BACf;4BACA,QAAQ;gCACN,MAAM;oCAAC;oCAAU;iCAAO;gCACxB,YAAY;oCACV,IAAI;wCAAE,MAAM;oCAAS;oCACrB,QAAQ;wCAAE,MAAM;oCAAS;gCAC3B;gCACA,UAAU;oCAAC;oCAAM;iCAAS;gCAC1B,sBAAsB;gCACtB,aAAa;4BACf;wBACF;wBACA,UAAU;4BAAC;4BAAW;4BAAQ;4BAAS;yBAAS;wBAChD,sBAAsB;oBACxB;oBACA,aAAa;gBACf;gBACA,oCAAoC;oBAClC,MAAM;oBACN,aACE;gBACJ;gBACA,KAAK;oBACH,MAAM;oBACN,aACE;gBACJ;gBACA,OAAO;oBACL,MAAM;wBAAC;wBAAU;qBAAO;oBACxB,aAAa;gBACf;YACF;YACA,UAAU;gBACR;gBACA;gBACA;gBACA;aACD;YACD,sBAAsB;QACxB;IACF;AACF"}

package/dist/lib/ai-model/prompt/llm-section-locator.js

@@ -68,10 +68,10 @@ If the description is "delete button on the second row with title 'Peter'", retu
 const sectionLocatorInstruction = (sectionDescription)=>`Find section containing: ${sectionDescription}`;
 exports.sectionLocatorInstruction = __webpack_exports__.sectionLocatorInstruction;
 exports.systemPromptToLocateSection = __webpack_exports__.systemPromptToLocateSection;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "sectionLocatorInstruction",
     "systemPromptToLocateSection"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/order-sensitive-judge.js

@@ -61,10 +61,10 @@ Return true if the description is order-sensitive, false otherwise.
 const orderSensitiveJudgePrompt = (description)=>`Analyze this element description: "${description}"`;
 exports.orderSensitiveJudgePrompt = __webpack_exports__.orderSensitiveJudgePrompt;
 exports.systemPromptToJudgeOrderSensitive = __webpack_exports__.systemPromptToJudgeOrderSensitive;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "orderSensitiveJudgePrompt",
     "systemPromptToJudgeOrderSensitive"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/playwright-generator.js

@@ -159,7 +159,7 @@ exports.getScreenshotsForLLM = __webpack_exports__.getScreenshotsForLLM;
 exports.prepareEventSummary = __webpack_exports__.prepareEventSummary;
 exports.processEventsForLLM = __webpack_exports__.processEventsForLLM;
 exports.validateEvents = __webpack_exports__.validateEvents;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "createEventCounts",
     "createMessageContent",
     "extractInputDescriptions",
@@ -170,7 +170,7 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "prepareEventSummary",
     "processEventsForLLM",
     "validateEvents"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/ui-tars-locator.js

@@ -58,9 +58,9 @@ call_user() # Submit the task and call the user when the task is unsolvable, or
     `;
 }
 exports.systemPromptToLocateElementPosition = __webpack_exports__.systemPromptToLocateElementPosition;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "systemPromptToLocateElementPosition"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/ui-tars-planning.js

@@ -62,10 +62,10 @@ finished(content='xxx') # Use escape characters \\', \\", and \\n in content par
 const getSummary = (prediction)=>prediction.replace(/Reflection:[\s\S]*?(?=Action_Summary:|Action:|$)/g, '').trim();
 exports.getSummary = __webpack_exports__.getSummary;
 exports.getUiTarsPlanningPrompt = __webpack_exports__.getUiTarsPlanningPrompt;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "getSummary",
     "getUiTarsPlanningPrompt"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/util.js

@@ -73,14 +73,14 @@ exports.describeUserPage = __webpack_exports__.describeUserPage;
 exports.distance = __webpack_exports__.distance;
 exports.distanceThreshold = __webpack_exports__.distanceThreshold;
 exports.samplePageDescription = __webpack_exports__.samplePageDescription;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "describeElement",
     "describeSize",
     "describeUserPage",
     "distance",
     "distanceThreshold",
     "samplePageDescription"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/prompt/yaml-generator.js

@@ -261,7 +261,7 @@ exports.getScreenshotsForLLM = __webpack_exports__.getScreenshotsForLLM;
 exports.prepareEventSummary = __webpack_exports__.prepareEventSummary;
 exports.processEventsForLLM = __webpack_exports__.processEventsForLLM;
 exports.validateEvents = __webpack_exports__.validateEvents;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "createEventCounts",
     "createMessageContent",
     "extractInputDescriptions",
@@ -272,7 +272,7 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "prepareEventSummary",
     "processEventsForLLM",
     "validateEvents"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/ai-model/service-caller/index.js

@@ -38,78 +38,37 @@ __webpack_require__.d(__webpack_exports__, {
     preprocessDoubaoBboxJson: ()=>preprocessDoubaoBboxJson,
     callAIWithStringResponse: ()=>callAIWithStringResponse,
     safeParseJson: ()=>safeParseJson,
+    getResponseFormat: ()=>getResponseFormat,
     callAI: ()=>callAI
 });
+const external_types_js_namespaceObject = require("../../types.js");
 const env_namespaceObject = require("@midscene/shared/env");
 const logger_namespaceObject = require("@midscene/shared/logger");
 const utils_namespaceObject = require("@midscene/shared/utils");
+const external_https_proxy_agent_namespaceObject = require("https-proxy-agent");
 const external_jsonrepair_namespaceObject = require("jsonrepair");
 const external_openai_namespaceObject = require("openai");
 var external_openai_default = /*#__PURE__*/ __webpack_require__.n(external_openai_namespaceObject);
+const external_socks_proxy_agent_namespaceObject = require("socks-proxy-agent");
+const external_common_js_namespaceObject = require("../../common.js");
+const assertion_js_namespaceObject = require("../prompt/assertion.js");
+const llm_planning_js_namespaceObject = require("../prompt/llm-planning.js");
 async function createChatClient({ AIActionTypeValue, modelConfig }) {
     const { socksProxy, httpProxy, modelName, openaiBaseURL, openaiApiKey, openaiExtraConfig, modelDescription, uiTarsModelVersion: uiTarsVersion, vlMode, createOpenAIClient, timeout } = modelConfig;
     let proxyAgent;
     const debugProxy = (0, logger_namespaceObject.getDebug)('ai:call:proxy');
-    const sanitizeProxyUrl = (url)=>{
-        try {
-            const parsed = new URL(url);
-            if (parsed.username) {
-                parsed.password = '****';
-                return parsed.href;
-            }
-            return url;
-        } catch  {
-            return url;
-        }
-    };
     if (httpProxy) {
-        debugProxy('using http proxy', sanitizeProxyUrl(httpProxy));
-        if (utils_namespaceObject.ifInBrowser) console.warn('HTTP proxy is configured but not supported in browser environment');
-        else {
-            const moduleName = 'undici';
-            const { ProxyAgent } = await import(moduleName);
-            proxyAgent = new ProxyAgent({
-                uri: httpProxy
-            });
-        }
+        debugProxy('using http proxy', httpProxy);
+        proxyAgent = new external_https_proxy_agent_namespaceObject.HttpsProxyAgent(httpProxy);
     } else if (socksProxy) {
-        debugProxy('using socks proxy', sanitizeProxyUrl(socksProxy));
-        if (utils_namespaceObject.ifInBrowser) console.warn('SOCKS proxy is configured but not supported in browser environment');
-        else try {
-            const moduleName = 'fetch-socks';
-            const { socksDispatcher } = await import(moduleName);
-            const proxyUrl = new URL(socksProxy);
-            if (!proxyUrl.hostname) throw new Error('SOCKS proxy URL must include a valid hostname');
-            const port = Number.parseInt(proxyUrl.port, 10);
-            if (!proxyUrl.port || Number.isNaN(port)) throw new Error('SOCKS proxy URL must include a valid port');
-            const protocol = proxyUrl.protocol.replace(':', '');
-            const socksType = 'socks4' === protocol ? 4 : 'socks5' === protocol ? 5 : 5;
-            proxyAgent = socksDispatcher({
-                type: socksType,
-                host: proxyUrl.hostname,
-                port,
-                ...proxyUrl.username ? {
-                    userId: decodeURIComponent(proxyUrl.username),
-                    password: decodeURIComponent(proxyUrl.password || '')
-                } : {}
-            });
-            debugProxy('socks proxy configured successfully', {
-                type: socksType,
-                host: proxyUrl.hostname,
-                port: port
-            });
-        } catch (error) {
-            console.error('Failed to configure SOCKS proxy:', error);
-            throw new Error(`Invalid SOCKS proxy URL: ${socksProxy}. Expected format: socks4://host:port, socks5://host:port, or with authentication: socks5://user:pass@host:port`);
-        }
+        debugProxy('using socks proxy', socksProxy);
+        proxyAgent = new external_socks_proxy_agent_namespaceObject.SocksProxyAgent(socksProxy);
     }
     const openAIOptions = {
         baseURL: openaiBaseURL,
         apiKey: openaiApiKey,
         ...proxyAgent ? {
-            fetchOptions: {
-                dispatcher: proxyAgent
-            }
+            httpAgent: proxyAgent
         } : {},
         ...openaiExtraConfig,
         ...'number' == typeof timeout ? {
@@ -150,6 +109,7 @@ async function callAI(messages, AIActionTypeValue, modelConfig, options) {
         AIActionTypeValue,
         modelConfig
     });
+    const responseFormat = getResponseFormat(modelName, AIActionTypeValue);
     const maxTokens = env_namespaceObject.globalConfigManager.getEnvConfigValue(env_namespaceObject.MIDSCENE_MODEL_MAX_TOKENS) ?? env_namespaceObject.globalConfigManager.getEnvConfigValue(env_namespaceObject.OPENAI_MAX_TOKENS);
     const debugCall = (0, logger_namespaceObject.getDebug)('ai:call');
     const debugProfileStats = (0, logger_namespaceObject.getDebug)('ai:profile:stats');
@@ -188,6 +148,7 @@ async function callAI(messages, AIActionTypeValue, modelConfig, options) {
             const stream = await completion.create({
                 model: modelName,
                 messages,
+                response_format: responseFormat,
                 ...commonConfig
             }, {
                 stream: true
@@ -234,6 +195,7 @@ async function callAI(messages, AIActionTypeValue, modelConfig, options) {
             const result = await completion.create({
                 model: modelName,
                 messages,
+                response_format: responseFormat,
                 ...commonConfig
             });
             timeCost = Date.now() - startTime;
@@ -266,6 +228,30 @@ async function callAI(messages, AIActionTypeValue, modelConfig, options) {
         throw newError;
     }
 }
+const getResponseFormat = (modelName, AIActionTypeValue)=>{
+    let responseFormat;
+    if (modelName.includes('gpt-4')) switch(AIActionTypeValue){
+        case external_common_js_namespaceObject.AIActionType.ASSERT:
+            responseFormat = assertion_js_namespaceObject.assertSchema;
+            break;
+        case external_common_js_namespaceObject.AIActionType.PLAN:
+            responseFormat = llm_planning_js_namespaceObject.planSchema;
+            break;
+        case external_common_js_namespaceObject.AIActionType.EXTRACT_DATA:
+        case external_common_js_namespaceObject.AIActionType.DESCRIBE_ELEMENT:
+            responseFormat = {
+                type: external_types_js_namespaceObject.AIResponseFormat.JSON
+            };
+            break;
+        case external_common_js_namespaceObject.AIActionType.TEXT:
+            responseFormat = void 0;
+            break;
+    }
+    if ('gpt-4o-2024-05-13' === modelName && AIActionTypeValue !== external_common_js_namespaceObject.AIActionType.TEXT) responseFormat = {
+        type: external_types_js_namespaceObject.AIResponseFormat.JSON
+    };
+    return responseFormat;
+};
 async function callAIWithObjectResponse(messages, AIActionTypeValue, modelConfig) {
     const response = await callAI(messages, AIActionTypeValue, modelConfig);
     (0, utils_namespaceObject.assert)(response, 'empty response');
@@ -338,16 +324,18 @@ exports.callAI = __webpack_exports__.callAI;
 exports.callAIWithObjectResponse = __webpack_exports__.callAIWithObjectResponse;
 exports.callAIWithStringResponse = __webpack_exports__.callAIWithStringResponse;
 exports.extractJSONFromCodeBlock = __webpack_exports__.extractJSONFromCodeBlock;
+exports.getResponseFormat = __webpack_exports__.getResponseFormat;
 exports.preprocessDoubaoBboxJson = __webpack_exports__.preprocessDoubaoBboxJson;
 exports.safeParseJson = __webpack_exports__.safeParseJson;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "callAI",
     "callAIWithObjectResponse",
     "callAIWithStringResponse",
     "extractJSONFromCodeBlock",
+    "getResponseFormat",
     "preprocessDoubaoBboxJson",
     "safeParseJson"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });