npm - @midscene/core - Versions diffs - 1.0.1-beta-20251217082334.0 → 1.0.1 - Mend

@midscene/core 1.0.1-beta-20251217082334.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (29) hide show

package/dist/es/agent/agent.mjs +2 -2
package/dist/es/agent/agent.mjs.map +1 -1
package/dist/es/agent/utils.mjs +1 -1
package/dist/es/ai-model/prompt/llm-planning.mjs +1 -1
package/dist/es/ai-model/prompt/llm-planning.mjs.map +1 -1
package/dist/es/common.mjs +2 -1
package/dist/es/common.mjs.map +1 -1
package/dist/es/index.mjs.map +1 -1
package/dist/es/utils.mjs +4 -3
package/dist/es/utils.mjs.map +1 -1
package/dist/es/yaml/utils.mjs +2 -3
package/dist/es/yaml/utils.mjs.map +1 -1
package/dist/lib/agent/agent.js +2 -2
package/dist/lib/agent/agent.js.map +1 -1
package/dist/lib/agent/utils.js +1 -1
package/dist/lib/ai-model/prompt/llm-planning.js +1 -1
package/dist/lib/ai-model/prompt/llm-planning.js.map +1 -1
package/dist/lib/common.js +2 -1
package/dist/lib/common.js.map +1 -1
package/dist/lib/index.js.map +1 -1
package/dist/lib/utils.js +4 -3
package/dist/lib/utils.js.map +1 -1
package/dist/lib/yaml/utils.js +2 -3
package/dist/lib/yaml/utils.js.map +1 -1
package/dist/types/agent/agent.d.ts +2 -5
package/dist/types/agent/index.d.ts +1 -0
package/dist/types/index.d.ts +1 -1
package/dist/types/yaml/utils.d.ts +1 -3
package/package.json +2 -2

package/dist/lib/ai-model/prompt/llm-planning.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/types';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport {\n getZodDescription,\n getZodTypeName,\n} from '@midscene/shared/zod-schema-utils';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\nimport type { z } from 'zod';\nimport { bboxDescription } from './common';\n\n// Note: put the log field first to trigger the CoT\n\nconst commonOutputFields = `\"error\"?: string, // Error messages about unexpected situations, if any. Only think it is an error when the situation is not foreseeable according to the instruction. Use the same language as the user's instruction.\n \"more_actions_needed_by_instruction\": boolean, // Consider if there is still more action(s) to do after the action in \"Log\" is done, according to the instruction. If so, set this field to true. Otherwise, set it to false.`;\n\nconst vlLocateParam = (vlMode: TVlModeTypes \| undefined) => {\n if (vlMode) {\n return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(vlMode)}`;\n }\n return '{ prompt: string /* description of the target element / }';\n};\n\nexport const descriptionForAction = (\n action: DeviceAction<any>,\n locatorSchemaTypeDescription: string,\n) => {\n const tab = ' ';\n const fields: string[] = [];\n\n // Add the action type field\n fields.push(`- type: \"${action.name}\"`);\n\n // Handle paramSchema if it exists\n if (action.paramSchema) {\n const paramLines: string[] = [];\n\n // Check if paramSchema is a ZodObject with shape\n const schema = action.paramSchema as {\n _def?: { typeName?: string };\n shape?: Record<string, unknown>;\n };\n const isZodObject = schema._def?.typeName === 'ZodObject';\n\n if (isZodObject && schema.shape) {\n // Original logic for ZodObject schemas\n const shape = schema.shape;\n\n for (const [key, field] of Object.entries(shape)) {\n if (field && typeof field === 'object') {\n // Check if field is optional\n const isOptional =\n typeof (field as { isOptional?: () => boolean }).isOptional ===\n 'function' &&\n (field as { isOptional: () => boolean }).isOptional();\n const keyWithOptional = isOptional ? `${key}?` : key;\n\n // Get the type name using extracted helper\n const typeName = getZodTypeName(field, locatorSchemaTypeDescription);\n\n // Get description using extracted helper\n const description = getZodDescription(field as z.ZodTypeAny);\n\n // Build param line for this field\n let paramLine = `${keyWithOptional}: ${typeName}`;\n if (description) {\n paramLine += ` // ${description}`;\n }\n\n paramLines.push(paramLine);\n }\n }\n\n // Add the param section to fields if there are paramLines\n if (paramLines.length > 0) {\n fields.push('- param:');\n paramLines.forEach((line) => {\n fields.push(` - ${line}`);\n });\n }\n } else {\n // Handle non-object schemas (string, number, etc.)\n const typeName = getZodTypeName(schema);\n const description = getZodDescription(schema as z.ZodTypeAny);\n\n // For simple types, indicate that param should be the direct value, not an object\n let paramDescription = `- param: ${typeName}`;\n if (description) {\n paramDescription += ` // ${description}`;\n }\n paramDescription += ' (pass the value directly, not as an object)';\n\n fields.push(paramDescription);\n }\n }\n\n return `- ${action.name}, ${action.description \|\| 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n actionSpace,\n vlMode,\n includeBbox,\n}: {\n actionSpace: DeviceAction<any>[];\n vlMode: TVlModeTypes \| undefined;\n includeBbox: boolean;\n}) {\n // Validate parameters: if includeBbox is true, vlMode must be defined\n if (includeBbox && !vlMode) {\n throw new Error(\n 'vlMode cannot be undefined when includeBbox is true. A valid vlMode is required for bbox-based location.',\n );\n }\n\n const actionDescriptionList = actionSpace.map((action) => {\n return descriptionForAction(\n action,\n vlLocateParam(includeBbox ? vlMode : undefined),\n );\n });\n const actionList = actionDescriptionList.join('\\n');\n\n const logFieldInstruction = `\n## About the \\`log\\` field (preamble message)\n\nThe \\`log\\` field is a brief preamble message to the user explaining what you’re about to do. It should follow these principles and examples:\n\n- Use the same language as the user's instruction\n- Keep it concise: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- Build on prior context: if this is not the first action to be done, use the preamble message to connect the dots with what’s been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- Keep your tone light, friendly and curious: add small touches of personality in preambles feel collaborative and engaging.\n\nExamples:*\n- \"Click the login button\"\n- \"Scroll to find the 'Yes' button in popup\"\n- \"Previous actions failed to find the 'Yes' button, i will try again\"\n- \"Go back to find the login button\"\n`;\n\n return `\nTarget: User will give you an instruction, some screenshots and previous logs indicating what have been done. Your task is to plan the next one action according to current situation to accomplish the instruction.\n\nPlease tell what the next one action is (or null if no action should be done) to do the tasks the instruction requires. \n\n## Rules\n\n- Don't give extra actions or plans beyond the instruction. For example, don't try to submit the form if the instruction is only to fill something.\n- Give just the next ONE action you should do\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully before performing the next step\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 5 times, you should think this is an error and set the \"error\" field to the error message.\n- If there is nothing to do but waiting, set the \"sleep\" field to the positive waiting time in milliseconds and null for the \"action\" field.\n- Assertions are also important steps. When getting the assertion instruction, a solid conclusion is required. You should explicitly state your conclusion by calling the \"Print_Assert_Result\" action.\n\n## Supporting actions\n${actionList}\n\n${logFieldInstruction}\n\n## Return format\n\nReturn in JSON format:\n{\n \"log\": string, // a brief preamble to the user explaining what you’re about to do\n ${commonOutputFields}\n \"action\": \n {\n \"type\": string, // the type of the action\n \"param\"?: { // The parameter of the action, if any\n // k-v style parameter fields\n }, \n } \| null,\n ,\n \"sleep\"?: number, // The sleep time after the action, in milliseconds.\n}\n\nFor example, if the instruction is to login and the form has already been filled, this is a valid return value:\n\n{\n \"log\": \"Click the login button\",\n \"more_actions_needed_by_instruction\": false,\n \"action\": {\n \"type\": \"Tap\",\n \"param\": {\n \"locate\": { \n \"prompt\": \"The login button\"${vlMode ? `, \"bbox\": [100, 200, 300, 400]` : ''}\n }\n }\n }\n`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","commonOutputFields","vlLocateParam","vlMode","bboxDescription","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","field","isOptional","keyWithOptional","typeName","getZodTypeName","description","getZodDescription","paramLine","line","paramDescription","systemPromptToTaskPlanning","actionSpace","includeBbox","Error","actionDescriptionList","undefined","actionList","logFieldInstruction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACMA,MAAMI,qBAAqB,CAAC;+NACmM,CAAC;AAEhO,MAAMC,gBAAgB,CAACC;IACrB,IAAIA,QACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EAAgBD,SAAS;IAElG,OAAO;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC;IAEA,MAAMC,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QAIjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAE1B,KAAK,MAAM,CAACf,KAAKkB,MAAM,IAAIjB,OAAO,OAAO,CAACgB,OACxC,IAAIC,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMC,aACJ,AACE,cADF,OAAQD,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBACrD,MAAME,kBAAkBD,aAAa,GAAGnB,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMqB,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAeJ,OAAOP;gBAGvC,MAAMY,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkBN;gBAGtC,IAAIO,YAAY,GAAGL,gBAAgB,EAAE,EAAEC,UAAU;gBACjD,IAAIE,aACFE,aAAa,CAAC,IAAI,EAAEF,aAAa;gBAGnCT,WAAW,IAAI,CAACW;YAClB;YAIF,IAAIX,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACY;oBAClBb,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEa,MAAM;gBAC3B;YACF;QACF,OAAO;YAEL,MAAML,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAeP;YAChC,MAAMQ,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkBT;YAGtC,IAAIY,mBAAmB,CAAC,SAAS,EAAEN,UAAU;YAC7C,IAAIE,aACFI,oBAAoB,CAAC,IAAI,EAAEJ,aAAa;YAE1CI,oBAAoB;YAEpBd,OAAO,IAAI,CAACc;QACd;IACF;IAEA,OAAO,CAAC,EAAE,EAAEjB,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAegB,2BAA2B,EAC/CC,WAAW,EACXtB,MAAM,EACNuB,WAAW,EAKZ;IAEC,IAAIA,eAAe,CAACvB,QAClB,MAAM,IAAIwB,MACR;IAIJ,MAAMC,wBAAwBH,YAAY,GAAG,CAAC,CAACnB,SACtCD,qBACLC,QACAJ,cAAcwB,cAAcvB,SAAS0B;IAGzC,MAAMC,aAAaF,sBAAsB,IAAI,CAAC;IAE9C,MAAMG,sBAAsB,CAAC;;;;;;;;;;;;;;;AAe/B,CAAC;IAEC,OAAO,CAAC;;;;;;;;;;;;;;;;AAgBV,EAAED,WAAW;;AAEb,EAAEC,oBAAoB;;;;;;;EAOpB,EAAE9B,mBAAmB;;;;;;;;;;;;;;;;;;;;;oCAqBa,EAAEE,SAAS,mCAAmC,GAAG;;;;AAIrF,CAAC;AACD"}
1	+ {"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DeviceAction } from '@/types';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport {\n getZodDescription,\n getZodTypeName,\n} from '@midscene/shared/zod-schema-utils';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\nimport type { z } from 'zod';\nimport { bboxDescription } from './common';\n\n// Note: put the log field first to trigger the CoT\n\nconst commonOutputFields = `\"error\"?: string, // Error messages about unexpected situations, if any. Only think it is an error when the situation is not foreseeable according to the instruction. Use the same language as the user's instruction.\n \"more_actions_needed_by_instruction\": boolean, // Consider if there is still more action(s) to do after the action in \"Log\" is done, according to the instruction. If so, set this field to true. Otherwise, set it to false.`;\n\nconst vlLocateParam = (vlMode: TVlModeTypes \| undefined) => {\n if (vlMode) {\n return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(vlMode)}`;\n }\n return '{ prompt: string /* description of the target element / }';\n};\n\nexport const descriptionForAction = (\n action: DeviceAction<any>,\n locatorSchemaTypeDescription: string,\n) => {\n const tab = ' ';\n const fields: string[] = [];\n\n // Add the action type field\n fields.push(`- type: \"${action.name}\"`);\n\n // Handle paramSchema if it exists\n if (action.paramSchema) {\n const paramLines: string[] = [];\n\n // Check if paramSchema is a ZodObject with shape\n const schema = action.paramSchema as {\n _def?: { typeName?: string };\n shape?: Record<string, unknown>;\n };\n const isZodObject = schema._def?.typeName === 'ZodObject';\n\n if (isZodObject && schema.shape) {\n // Original logic for ZodObject schemas\n const shape = schema.shape;\n\n for (const [key, field] of Object.entries(shape)) {\n if (field && typeof field === 'object') {\n // Check if field is optional\n const isOptional =\n typeof (field as { isOptional?: () => boolean }).isOptional ===\n 'function' &&\n (field as { isOptional: () => boolean }).isOptional();\n const keyWithOptional = isOptional ? `${key}?` : key;\n\n // Get the type name using extracted helper\n const typeName = getZodTypeName(field, locatorSchemaTypeDescription);\n\n // Get description using extracted helper\n const description = getZodDescription(field as z.ZodTypeAny);\n\n // Build param line for this field\n let paramLine = `${keyWithOptional}: ${typeName}`;\n if (description) {\n paramLine += ` // ${description}`;\n }\n\n paramLines.push(paramLine);\n }\n }\n\n // Add the param section to fields if there are paramLines\n if (paramLines.length > 0) {\n fields.push('- param:');\n paramLines.forEach((line) => {\n fields.push(` - ${line}`);\n });\n }\n } else {\n // Handle non-object schemas (string, number, etc.)\n const typeName = getZodTypeName(schema);\n const description = getZodDescription(schema as z.ZodTypeAny);\n\n // For simple types, indicate that param should be the direct value, not an object\n let paramDescription = `- param: ${typeName}`;\n if (description) {\n paramDescription += ` // ${description}`;\n }\n paramDescription += ' (pass the value directly, not as an object)';\n\n fields.push(paramDescription);\n }\n }\n\n return `- ${action.name}, ${action.description \|\| 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n actionSpace,\n vlMode,\n includeBbox,\n}: {\n actionSpace: DeviceAction<any>[];\n vlMode: TVlModeTypes \| undefined;\n includeBbox: boolean;\n}) {\n // Validate parameters: if includeBbox is true, vlMode must be defined\n if (includeBbox && !vlMode) {\n throw new Error(\n 'vlMode cannot be undefined when includeBbox is true. A valid vlMode is required for bbox-based location.',\n );\n }\n\n const actionDescriptionList = actionSpace.map((action) => {\n return descriptionForAction(\n action,\n vlLocateParam(includeBbox ? vlMode : undefined),\n );\n });\n const actionList = actionDescriptionList.join('\\n');\n\n const logFieldInstruction = `\n## About the \\`log\\` field (preamble message)\n\nThe \\`log\\` field is a brief preamble message to the user explaining what you’re about to do. It should follow these principles and examples:\n\n- Use the same language as the user's instruction\n- Keep it concise: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- Build on prior context: if this is not the first action to be done, use the preamble message to connect the dots with what’s been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- Keep your tone light, friendly and curious: add small touches of personality in preambles feel collaborative and engaging.\n\nExamples:*\n- \"Click the login button\"\n- \"Scroll to find the 'Yes' button in popup\"\n- \"Previous actions failed to find the 'Yes' button, i will try again\"\n- \"Go back to find the login button\"\n`;\n\n return `\nTarget: User will give you an instruction, some screenshots and previous logs indicating what have been done. Your task is to plan the next one action according to current situation to accomplish the instruction.\n\nPlease tell what the next one action is (or null if no action should be done) to do the tasks the instruction requires. \n\n## Rules\n\n- Don't give extra actions or plans beyond the instruction. For example, don't try to submit the form if the instruction is only to fill something.\n- Give just the next ONE action you should do\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully before performing the next step\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 5 times, you should think this is an error and set the \"error\" field to the error message.\n- If there is nothing to do but waiting, set the \"sleep\" field to the positive waiting time in milliseconds and null for the \"action\" field.\n- Assertions are also important steps. When getting the assertion instruction, a solid conclusion is required. You should explicitly state your conclusion by calling the \"Print_Assert_Result\" action.\n\n## Supporting actions\n${actionList}\n\n${logFieldInstruction}\n\n## Return format\n\nReturn in JSON format:\n{\n \"log\": string, // a brief preamble to the user explaining what you’re about to do\n ${commonOutputFields}\n \"action\": \n {\n \"type\": string, // the type of the action\n \"param\"?: { // The parameter of the action, if any\n // k-v style parameter fields\n }, \n } \| null,\n ,\n \"sleep\"?: number, // The sleep time after the action, in milliseconds.\n}\n\nFor example, if the instruction is to login and the form has already been filled, this is a valid return value:\n\n{\n \"log\": \"Click the login button\",\n \"more_actions_needed_by_instruction\": false,\n \"action\": {\n \"type\": \"Tap\",\n \"param\": {\n \"locate\": { \n \"prompt\": \"The login button\"${vlMode && includeBbox ? `, \"bbox\": [100, 200, 300, 400]` : ''}\n }\n }\n }\n`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","commonOutputFields","vlLocateParam","vlMode","bboxDescription","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","field","isOptional","keyWithOptional","typeName","getZodTypeName","description","getZodDescription","paramLine","line","paramDescription","systemPromptToTaskPlanning","actionSpace","includeBbox","Error","actionDescriptionList","undefined","actionList","logFieldInstruction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACMA,MAAMI,qBAAqB,CAAC;+NACmM,CAAC;AAEhO,MAAMC,gBAAgB,CAACC;IACrB,IAAIA,QACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EAAgBD,SAAS;IAElG,OAAO;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC;IAEA,MAAMC,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QAIjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAE1B,KAAK,MAAM,CAACf,KAAKkB,MAAM,IAAIjB,OAAO,OAAO,CAACgB,OACxC,IAAIC,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMC,aACJ,AACE,cADF,OAAQD,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBACrD,MAAME,kBAAkBD,aAAa,GAAGnB,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMqB,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAeJ,OAAOP;gBAGvC,MAAMY,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkBN;gBAGtC,IAAIO,YAAY,GAAGL,gBAAgB,EAAE,EAAEC,UAAU;gBACjD,IAAIE,aACFE,aAAa,CAAC,IAAI,EAAEF,aAAa;gBAGnCT,WAAW,IAAI,CAACW;YAClB;YAIF,IAAIX,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACY;oBAClBb,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEa,MAAM;gBAC3B;YACF;QACF,OAAO;YAEL,MAAML,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAeP;YAChC,MAAMQ,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkBT;YAGtC,IAAIY,mBAAmB,CAAC,SAAS,EAAEN,UAAU;YAC7C,IAAIE,aACFI,oBAAoB,CAAC,IAAI,EAAEJ,aAAa;YAE1CI,oBAAoB;YAEpBd,OAAO,IAAI,CAACc;QACd;IACF;IAEA,OAAO,CAAC,EAAE,EAAEjB,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAegB,2BAA2B,EAC/CC,WAAW,EACXtB,MAAM,EACNuB,WAAW,EAKZ;IAEC,IAAIA,eAAe,CAACvB,QAClB,MAAM,IAAIwB,MACR;IAIJ,MAAMC,wBAAwBH,YAAY,GAAG,CAAC,CAACnB,SACtCD,qBACLC,QACAJ,cAAcwB,cAAcvB,SAAS0B;IAGzC,MAAMC,aAAaF,sBAAsB,IAAI,CAAC;IAE9C,MAAMG,sBAAsB,CAAC;;;;;;;;;;;;;;;AAe/B,CAAC;IAEC,OAAO,CAAC;;;;;;;;;;;;;;;;AAgBV,EAAED,WAAW;;AAEb,EAAEC,oBAAoB;;;;;;;EAOpB,EAAE9B,mBAAmB;;;;;;;;;;;;;;;;;;;;;oCAqBa,EAAEE,UAAUuB,cAAc,mCAAmC,GAAG;;;;AAIpG,CAAC;AACD"}

package/dist/lib/common.js CHANGED Viewed

@@ -204,7 +204,8 @@ function mergeRects(rects) {
     };
 }
 function expandSearchArea(rect, screenSize, vlMode) {
-    const minEdgeSize = 500;
+    let minEdgeSize = 500;
+    if ('qwen3-vl' === vlMode) minEdgeSize = 700;
     const defaultPadding = 160;
     const paddingSizeHorizontal = rect.width < minEdgeSize ? Math.ceil((minEdgeSize - rect.width) / 2) : defaultPadding;
     const paddingSizeVertical = rect.height < minEdgeSize ? Math.ceil((minEdgeSize - rect.height) / 2) : defaultPadding;

package/dist/lib/common.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"common.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/common.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n BaseElement,\n DeviceAction,\n ElementTreeNode,\n MidsceneYamlFlowItem,\n PlanningAction,\n Rect,\n Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nexport enum AIActionType {\n ASSERT = 0,\n INSPECT_ELEMENT = 1,\n EXTRACT_DATA = 2,\n PLAN = 3,\n DESCRIBE_ELEMENT = 4,\n TEXT = 5,\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\ntype AdaptBboxInput = number[] \| string[] \| string \| (number[] \| string[])[];\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n locate: PlanningLocateParam,\n width: number,\n height: number,\n rightLimit: number,\n bottomLimit: number,\n vlMode: TVlModeTypes \| undefined,\n) {\n // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n if ((locate as any).bbox_2d && !locate?.bbox) {\n locate.bbox = (locate as any).bbox_2d;\n // biome-ignore lint/performance/noDelete: <explanation>\n delete (locate as any).bbox_2d;\n }\n\n if (locate?.bbox) {\n locate.bbox = adaptBbox(\n locate.bbox,\n width,\n height,\n rightLimit,\n bottomLimit,\n vlMode,\n );\n }\n\n return locate;\n}\n\nexport function adaptQwen2_5Bbox(\n bbox: number[],\n): [number, number, number, number] {\n if (bbox.length < 2) {\n const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n throw new Error(msg);\n }\n\n const result: [number, number, number, number] = [\n Math.round(bbox[0]),\n Math.round(bbox[1]),\n typeof bbox[2] === 'number'\n ? Math.round(bbox[2])\n : Math.round(bbox[0] + defaultBboxSize),\n typeof bbox[3] === 'number'\n ? Math.round(bbox[3])\n : Math.round(bbox[1] + defaultBboxSize),\n ];\n return result;\n}\n\nexport function adaptDoubaoBbox(\n bbox: string[] \| number[] \| string,\n width: number,\n height: number,\n): [number, number, number, number] {\n assert(\n width > 0 && height > 0,\n 'width and height must be greater than 0 in doubao mode',\n );\n\n if (typeof bbox === 'string') {\n assert(\n /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n `invalid bbox data string for doubao-vision mode: ${bbox}`,\n );\n const splitted = bbox.split(' ');\n if (splitted.length === 4) {\n return [\n Math.round((Number(splitted[0]) * width) / 1000),\n Math.round((Number(splitted[1]) * height) / 1000),\n Math.round((Number(splitted[2]) * width) / 1000),\n Math.round((Number(splitted[3]) * height) / 1000),\n ];\n }\n throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n }\n\n let bboxList: number[] = [];\n if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n bbox.forEach((item) => {\n if (typeof item === 'string' && item.includes(',')) {\n const [x, y] = item.split(',');\n bboxList.push(Number(x.trim()), Number(y.trim()));\n } else if (typeof item === 'string' && item.includes(' ')) {\n const [x, y] = item.split(' ');\n bboxList.push(Number(x.trim()), Number(y.trim()));\n } else {\n bboxList.push(Number(item));\n }\n });\n } else {\n bboxList = bbox as any;\n }\n\n if (bboxList.length === 4 \|\| bboxList.length === 5) {\n return [\n Math.round((bboxList[0] * width) / 1000),\n Math.round((bboxList[1] * height) / 1000),\n Math.round((bboxList[2] * width) / 1000),\n Math.round((bboxList[3] * height) / 1000),\n ];\n }\n\n // treat the bbox as a center point\n if (\n bboxList.length === 6 \|\|\n bboxList.length === 2 \|\|\n bboxList.length === 3 \|\|\n bboxList.length === 7\n ) {\n return [\n Math.max(\n 0,\n Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n ),\n Math.max(\n 0,\n Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n ),\n Math.min(\n width,\n Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n ),\n Math.min(\n height,\n Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n ),\n ];\n }\n\n if (bbox.length === 8) {\n return [\n Math.round((bboxList[0] * width) / 1000),\n Math.round((bboxList[1] * height) / 1000),\n Math.round((bboxList[4] * width) / 1000),\n Math.round((bboxList[5] * height) / 1000),\n ];\n }\n\n const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n throw new Error(msg);\n}\n\nfunction normalizeBboxInput(\n bbox: AdaptBboxInput,\n): number[] \| string[] \| string {\n if (Array.isArray(bbox)) {\n if (Array.isArray(bbox[0])) {\n return bbox[0] as number[] \| string[];\n }\n return bbox as number[] \| string[];\n }\n return bbox as string;\n}\n\nexport function adaptBbox(\n bbox: AdaptBboxInput,\n width: number,\n height: number,\n rightLimit: number,\n bottomLimit: number,\n vlMode: TVlModeTypes \| undefined,\n): [number, number, number, number] {\n const normalizedBbox = normalizeBboxInput(bbox);\n\n let result: [number, number, number, number] = [0, 0, 0, 0];\n if (vlMode === 'doubao-vision' \|\| vlMode === 'vlm-ui-tars') {\n result = adaptDoubaoBbox(normalizedBbox, width, height);\n } else if (vlMode === 'gemini') {\n result = adaptGeminiBbox(normalizedBbox as number[], width, height);\n } else if (vlMode === 'qwen3-vl') {\n result = normalized01000(normalizedBbox as number[], width, height);\n } else {\n result = adaptQwen2_5Bbox(normalizedBbox as number[]);\n }\n\n result[2] = Math.min(result[2], rightLimit);\n result[3] = Math.min(result[3], bottomLimit);\n\n return result;\n}\n\n// x1, y1, x2, y2 -> 0-1000\nexport function normalized01000(\n bbox: number[],\n width: number,\n height: number,\n): [number, number, number, number] {\n return [\n Math.round((bbox[0] * width) / 1000),\n Math.round((bbox[1] * height) / 1000),\n Math.round((bbox[2] * width) / 1000),\n Math.round((bbox[3] * height) / 1000),\n ];\n}\n\n// y1, x1, y2, x2 -> 0-1000\nexport function adaptGeminiBbox(\n bbox: number[],\n width: number,\n height: number,\n): [number, number, number, number] {\n const left = Math.round((bbox[1] * width) / 1000);\n const top = Math.round((bbox[0] * height) / 1000);\n const right = Math.round((bbox[3] * width) / 1000);\n const bottom = Math.round((bbox[2] * height) / 1000);\n return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n bbox: number[],\n width: number,\n height: number,\n offsetX = 0,\n offsetY = 0,\n rightLimit = width,\n bottomLimit = height,\n vlMode?: TVlModeTypes \| undefined,\n): Rect {\n debugInspectUtils(\n 'adaptBboxToRect',\n bbox,\n width,\n height,\n 'offset',\n offsetX,\n offsetY,\n 'limit',\n rightLimit,\n bottomLimit,\n 'vlMode',\n vlMode,\n );\n const [left, top, right, bottom] = adaptBbox(\n bbox,\n width,\n height,\n rightLimit,\n bottomLimit,\n vlMode,\n );\n\n // Calculate initial rect dimensions\n const rectLeft = left;\n const rectTop = top;\n let rectWidth = right - left;\n let rectHeight = bottom - top;\n\n // Ensure the rect doesn't exceed image boundaries\n // If right edge exceeds width, adjust the width\n if (rectLeft + rectWidth > width) {\n rectWidth = width - rectLeft;\n }\n\n // If bottom edge exceeds height, adjust the height\n if (rectTop + rectHeight > height) {\n rectHeight = height - rectTop;\n }\n\n // Ensure minimum dimensions (width and height should be at least 1)\n rectWidth = Math.max(1, rectWidth);\n rectHeight = Math.max(1, rectHeight);\n\n const rect = {\n left: rectLeft + offsetX,\n top: rectTop + offsetY,\n width: rectWidth,\n height: rectHeight,\n };\n debugInspectUtils('adaptBboxToRect, result=', rect);\n\n return rect;\n}\n\nexport function mergeRects(rects: Rect[]) {\n const minLeft = Math.min(...rects.map((r) => r.left));\n const minTop = Math.min(...rects.map((r) => r.top));\n const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n return {\n left: minLeft,\n top: minTop,\n width: maxRight - minLeft,\n height: maxBottom - minTop,\n };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n rect: Rect,\n screenSize: Size,\n vlMode: TVlModeTypes \| undefined,\n) {\n const minEdgeSize = 500; // vlMode === 'doubao-vision' ? 500 : 300;\n const defaultPadding = 160;\n\n // Calculate padding needed to reach minimum edge size\n const paddingSizeHorizontal =\n rect.width < minEdgeSize\n ? Math.ceil((minEdgeSize - rect.width) / 2)\n : defaultPadding;\n const paddingSizeVertical =\n rect.height < minEdgeSize\n ? Math.ceil((minEdgeSize - rect.height) / 2)\n : defaultPadding;\n\n // Calculate new dimensions (ensure minimum edge size)\n let newWidth = Math.max(minEdgeSize, rect.width + paddingSizeHorizontal * 2);\n let newHeight = Math.max(minEdgeSize, rect.height + paddingSizeVertical * 2);\n\n // Calculate initial position with padding\n let newLeft = rect.left - paddingSizeHorizontal;\n let newTop = rect.top - paddingSizeVertical;\n\n // Ensure the rect doesn't exceed screen boundaries by adjusting position\n // If the rect goes beyond the right edge, shift it left\n if (newLeft + newWidth > screenSize.width) {\n newLeft = screenSize.width - newWidth;\n }\n\n // If the rect goes beyond the bottom edge, shift it up\n if (newTop + newHeight > screenSize.height) {\n newTop = screenSize.height - newHeight;\n }\n\n // Ensure the rect doesn't go beyond the left/top edges\n newLeft = Math.max(0, newLeft);\n newTop = Math.max(0, newTop);\n\n // If after position adjustment, the rect still exceeds screen boundaries,\n // clamp the dimensions to fit within screen\n if (newLeft + newWidth > screenSize.width) {\n newWidth = screenSize.width - newLeft;\n }\n if (newTop + newHeight > screenSize.height) {\n newHeight = screenSize.height - newTop;\n }\n\n rect.left = newLeft;\n rect.top = newTop;\n rect.width = newWidth;\n rect.height = newHeight;\n\n return rect;\n}\n\nexport async function markupImageForLLM(\n screenshotBase64: string,\n tree: ElementTreeNode<BaseElement>,\n size: Size,\n) {\n const elementsInfo = treeToList(tree);\n const elementsPositionInfoWithoutText = elementsInfo!.filter(\n (elementInfo) => {\n if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n return false;\n }\n return true;\n },\n );\n\n const imagePayload = await compositeElementInfoImg({\n inputImgBase64: screenshotBase64,\n elementsPositionInfo: elementsPositionInfoWithoutText,\n size,\n });\n return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n plans: PlanningAction[],\n actionSpace: DeviceAction<any>[],\n sleep?: number,\n): MidsceneYamlFlowItem[] {\n const flow: MidsceneYamlFlowItem[] = [];\n\n for (const plan of plans) {\n const verb = plan.type;\n\n const action = actionSpace.find((action) => action.name === verb);\n if (!action) {\n console.warn(\n `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n );\n continue;\n }\n\n const flowKey = action.interfaceAlias \|\| verb;\n const flowParam = action.paramSchema\n ? dumpActionParam(plan.param \|\| {}, action.paramSchema)\n : {};\n\n const flowItem: MidsceneYamlFlowItem = {\n [flowKey]: '',\n ...flowParam,\n };\n\n flow.push(flowItem);\n }\n\n if (sleep) {\n flow.push({\n sleep,\n });\n }\n\n return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n left: z.number(),\n top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n width: z.number(),\n height: z.number(),\n dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n z.object({\n zoom: z.number().optional(),\n }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n images: z\n .array(\n z.object({\n name: z.string(),\n url: z.string(),\n }),\n )\n .optional(),\n convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n z.string(),\n z\n .object({\n prompt: z.string(),\n })\n .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n .object({\n prompt: TUserPromptSchema,\n deepThink: z.boolean().optional(),\n cacheable: z.boolean().optional(),\n xpath: z.union([z.string(), z.boolean()]).optional(),\n })\n .passthrough();\n\n// Schema for locator field result (when AI returns locate results)\nconst MidsceneLocationResult = z\n .object({\n [locateFieldFlagName]: z.literal(true),\n prompt: TUserPromptSchema,\n\n // optional fields\n deepThink: z.boolean().optional(), // only available in vl model\n cacheable: z.boolean().optional(),\n xpath: z.boolean().optional(), // preset result for xpath\n\n // these two fields will only appear in the result\n center: z.tuple([z.number(), z.number()]),\n rect: RectSchema,\n })\n .passthrough();\n\n// Export the result type - this is used for runtime results that include center and rect\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\n\n// Export the input type - this is the inferred type from getMidsceneLocationSchema()\nexport type MidsceneLocationInputType = z.infer<typeof MidsceneLocationInput>;\n\n/*\n Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n /\nexport const getMidsceneLocationSchema = () => {\n return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n // Handle optional fields by getting the inner type\n let actualField = field;\n if (actualField._def?.typeName === 'ZodOptional') {\n actualField = actualField._def.innerType;\n }\n\n // Check if this is a ZodObject\n if (actualField._def?.typeName === 'ZodObject') {\n const shape = actualField._def.shape();\n\n // Method 1: Check for the location field flag (for result schema)\n if (locateFieldFlagName in shape) {\n return true;\n }\n\n // Method 2: Check if it's the input schema by checking for 'prompt' field\n // Input schema has 'prompt' as a required field\n if ('prompt' in shape && shape.prompt) {\n return true;\n }\n }\n\n return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n assert(\n ifMidsceneLocatorField(field),\n 'field is not a midscene locator field',\n );\n\n // If field is a string, return it directly\n if (typeof field === 'string') {\n return field;\n }\n\n // If field is an object with prompt property\n if (field && typeof field === 'object' && field.prompt) {\n // If prompt is a string, return it directly\n if (typeof field.prompt === 'string') {\n return field.prompt;\n }\n // If prompt is a TUserPrompt object, extract the prompt string\n if (typeof field.prompt === 'object' && field.prompt.prompt) {\n return field.prompt.prompt; // TODO: dump images if necessary\n }\n }\n\n // Fallback: try to convert to string\n return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n zodType?: z.ZodType<any>,\n requiredOnly?: boolean,\n): string[] => {\n if (!zodType) {\n return [];\n }\n\n // Check if this is a ZodObject by checking if it has a shape property\n const zodObject = zodType as any;\n if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n const keys = Object.keys(zodObject.shape);\n return keys.filter((key) => {\n const field = zodObject.shape[key];\n if (!ifMidsceneLocatorField(field)) {\n return false;\n }\n\n // If requiredOnly is true, filter out optional fields\n if (requiredOnly) {\n return field._def?.typeName !== 'ZodOptional';\n }\n\n return true;\n });\n }\n\n // For other ZodType instances, we can't extract field names\n return [];\n};\n\nexport const dumpActionParam = (\n jsonObject: Record<string, any>,\n zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n const locatorFields = findAllMidsceneLocatorField(zodSchema);\n const result = { ...jsonObject };\n\n for (const fieldName of locatorFields) {\n const fieldValue = result[fieldName];\n if (fieldValue) {\n // If it's already a string, keep it as is\n if (typeof fieldValue === 'string') {\n result[fieldName] = fieldValue;\n } else if (typeof fieldValue === 'object') {\n // Check if this field is actually a MidsceneLocationType object\n if (fieldValue.prompt) {\n // If prompt is a string, use it directly\n if (typeof fieldValue.prompt === 'string') {\n result[fieldName] = fieldValue.prompt;\n } else if (\n typeof fieldValue.prompt === 'object' &&\n fieldValue.prompt.prompt\n ) {\n // If prompt is a TUserPrompt object, extract the prompt string\n result[fieldName] = fieldValue.prompt.prompt;\n }\n }\n }\n }\n }\n\n return result;\n};\n\nexport const loadActionParam = (\n jsonObject: Record<string, any>,\n zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n const locatorFields = findAllMidsceneLocatorField(zodSchema);\n const result = { ...jsonObject };\n\n for (const fieldName of locatorFields) {\n const fieldValue = result[fieldName];\n if (fieldValue && typeof fieldValue === 'string') {\n result[fieldName] = {\n [locateFieldFlagName]: true,\n prompt: fieldValue,\n };\n }\n }\n\n return result;\n};\n\n/\n Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n \n Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n */\nexport const parseActionParam = (\n rawParam: Record<string, any> \| undefined,\n zodSchema?: z.ZodType<any>,\n): Record<string, any> \| undefined => {\n // If no schema is provided, return undefined (action takes no parameters)\n if (!zodSchema) {\n return undefined;\n }\n\n // Handle undefined or null rawParam by providing an empty object\n const param = rawParam ?? {};\n\n // Find all locate fields in the schema\n const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n // If there are no locate fields, just do normal validation\n if (locateFields.length === 0) {\n return zodSchema.parse(param);\n }\n\n // Extract locate field values to restore later\n const locateFieldValues: Record<string, any> = {};\n for (const fieldName of locateFields) {\n if (fieldName in param) {\n locateFieldValues[fieldName] = param[fieldName];\n }\n }\n\n // Build params for validation - skip locate fields and use dummy values\n const paramsForValidation: Record<string, any> = {};\n for (const key in param) {\n if (locateFields.includes(key)) {\n // Use dummy value to satisfy schema validation\n paramsForValidation[key] = { prompt: '_dummy_' };\n } else {\n paramsForValidation[key] = param[key];\n }\n }\n\n // Validate with dummy locate values\n const validated = zodSchema.parse(paramsForValidation);\n\n // Restore the actual locate field values (unvalidated, as per business requirement)\n for (const fieldName in locateFieldValues) {\n validated[fieldName] = locateFieldValues[fieldName];\n }\n\n return validated;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","AIActionType","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","rightLimit","bottomLimit","vlMode","adaptBbox","adaptQwen2_5Bbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","bboxList","Array","item","x","y","normalizeBboxInput","normalizedBbox","adaptGeminiBbox","normalized01000","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rectLeft","rectTop","rectWidth","rectHeight","rect","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","size","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","console","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","jsonObject","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam","parseActionParam","rawParam","param","locateFields","locateFieldValues","paramsForValidation","validated"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACiBO,IAAKI,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;;WAAZA;;AASZ,MAAMC,kBAAkB;AACxB,MAAMC,oBAAoBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAI5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAGhC,IAAKL,OAAe,OAAO,IAAI,CAACA,QAAQ,MAAM;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAQ,MACVA,OAAO,IAAI,GAAGM,UACZN,OAAO,IAAI,EACXC,OACAC,QACAC,YACAC,aACAC;IAIJ,OAAOL;AACT;AAEO,SAASO,iBACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;QACN,YAAnB,OAAOY,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;KAC1B;IACD,OAAOgB;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCP,KAAa,EACbC,MAAc;IAEda,IAAAA,sBAAAA,MAAAA,AAAAA,EACEd,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOM,MAAmB;QAC5BO,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;YAC5CW,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;SAC7C;QAEH,MAAM,IAAIS,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,WAAqB,EAAE;IAC3B,IAAIC,MAAM,OAAO,CAACX,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACD,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACD,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEJ,SAAS,IAAI,CAACD,OAAOG;IAEzB;SAEAF,WAAWV;IAGb,IAAIU,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLL,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;QACpCW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;KACrC;IAIH,IACEgB,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLL,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU,QAAQN,kBAAkB;QAEhEiB,KAAK,GAAG,CACNZ,OACAY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACNX,QACAW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIY,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;QACpCW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;KACrC;IAGH,MAAMO,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEA,SAASc,mBACPf,IAAoB;IAEpB,IAAIW,MAAM,OAAO,CAACX,OAChB;QAAA,IAAIW,MAAM,OAAO,CAACX,IAAI,CAAC,EAAE,GACvB,OAAOA,IAAI,CAAC,EAAE;IAChB;IAGF,OAAOA;AACT;AAEO,SAASF,UACdE,IAAoB,EACpBP,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAEhC,MAAMmB,iBAAiBD,mBAAmBf;IAE1C,IAAII,SAA2C;QAAC;QAAG;QAAG;QAAG;KAAE;IAEzDA,SADEP,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,SACvBS,gBAAgBU,gBAAgBvB,OAAOC,UACvCG,AAAW,aAAXA,SACAoB,gBAAgBD,gBAA4BvB,OAAOC,UACnDG,AAAW,eAAXA,SACAqB,gBAAgBF,gBAA4BvB,OAAOC,UAEnDK,iBAAiBiB;IAG5BZ,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAET;IAChCS,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAER;IAEhC,OAAOQ;AACT;AAGO,SAASc,gBACdlB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,OAAO;QACLW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;QAChCW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;KACjC;AACH;AAGO,SAASuB,gBACdjB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,MAAMyB,OAAOd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC5C,MAAM2B,MAAMf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC5C,MAAM2B,QAAQhB,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC7C,MAAM6B,SAASjB,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC/C,OAAO;QAACyB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdvB,IAAc,EACdP,KAAa,EACbC,MAAc,EACd8B,UAAU,CAAC,EACXC,UAAU,CAAC,EACX9B,aAAaF,KAAK,EAClBG,cAAcF,MAAM,EACpBG,MAAiC;IAEjCR,kBACE,mBACAW,MACAP,OACAC,QACA,UACA8B,SACAC,SACA,SACA9B,YACAC,aACA,UACAC;IAEF,MAAM,CAACsB,MAAMC,KAAKC,OAAOC,OAAO,GAAGxB,UACjCE,MACAP,OACAC,QACAC,YACAC,aACAC;IAIF,MAAM6B,WAAWP;IACjB,MAAMQ,UAAUP;IAChB,IAAIQ,YAAYP,QAAQF;IACxB,IAAIU,aAAaP,SAASF;IAI1B,IAAIM,WAAWE,YAAYnC,OACzBmC,YAAYnC,QAAQiC;IAItB,IAAIC,UAAUE,aAAanC,QACzBmC,aAAanC,SAASiC;IAIxBC,YAAYvB,KAAK,GAAG,CAAC,GAAGuB;IACxBC,aAAaxB,KAAK,GAAG,CAAC,GAAGwB;IAEzB,MAAMC,OAAO;QACX,MAAMJ,WAAWF;QACjB,KAAKG,UAAUF;QACf,OAAOG;QACP,QAAQC;IACV;IACAxC,kBAAkB,4BAA4ByC;IAE9C,OAAOA;AACT;AAEO,SAASC,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdR,IAAU,EACVS,UAAgB,EAChB1C,MAAgC;IAEhC,MAAM2C,cAAc;IACpB,MAAMC,iBAAiB;IAGvB,MAAMC,wBACJZ,KAAK,KAAK,GAAGU,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAcV,KAAK,KAAI,IAAK,KACvCW;IACN,MAAME,sBACJb,KAAK,MAAM,GAAGU,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAcV,KAAK,MAAK,IAAK,KACxCW;IAGN,IAAIG,WAAWvC,KAAK,GAAG,CAACmC,aAAaV,KAAK,KAAK,GAAGY,AAAwB,IAAxBA;IAClD,IAAIG,YAAYxC,KAAK,GAAG,CAACmC,aAAaV,KAAK,MAAM,GAAGa,AAAsB,IAAtBA;IAGpD,IAAIG,UAAUhB,KAAK,IAAI,GAAGY;IAC1B,IAAIK,SAASjB,KAAK,GAAG,GAAGa;IAIxB,IAAIG,UAAUF,WAAWL,WAAW,KAAK,EACvCO,UAAUP,WAAW,KAAK,GAAGK;IAI/B,IAAIG,SAASF,YAAYN,WAAW,MAAM,EACxCQ,SAASR,WAAW,MAAM,GAAGM;IAI/BC,UAAUzC,KAAK,GAAG,CAAC,GAAGyC;IACtBC,SAAS1C,KAAK,GAAG,CAAC,GAAG0C;IAIrB,IAAID,UAAUF,WAAWL,WAAW,KAAK,EACvCK,WAAWL,WAAW,KAAK,GAAGO;IAEhC,IAAIC,SAASF,YAAYN,WAAW,MAAM,EACxCM,YAAYN,WAAW,MAAM,GAAGQ;IAGlCjB,KAAK,IAAI,GAAGgB;IACZhB,KAAK,GAAG,GAAGiB;IACXjB,KAAK,KAAK,GAAGc;IACbd,KAAK,MAAM,GAAGe;IAEd,OAAOf;AACT;AAEO,eAAekB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCC,IAAU;IAEV,MAAMC,eAAeC,AAAAA,IAAAA,0BAAAA,UAAAA,AAAAA,EAAWH;IAChC,MAAMI,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,0BAAAA,QAAAA,CAAAA,IAAa,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;QACjD,gBAAgBT;QAChB,sBAAsBK;QACtBH;IACF;IACA,OAAOM;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEF,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMG,UAAUF,OAAO,cAAc,IAAID;QACzC,MAAMI,YAAYH,OAAO,WAAW,GAChCI,gBAAgBN,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMK,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAN,KAAK,IAAI,CAACQ;IACZ;IAEA,IAAIT,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMS,cAAcC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClC,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;IACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;AACf;AAEO,MAAMC,aAAaD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjC,OAAOA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;IAChB,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,QAAQA,6BAAAA,CAAAA,CAAAA,KACA,CACJA,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;QACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;QACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,IAED,QAAQ;IACX,yBAAyBA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;IACvCA,6BAAAA,CAAAA,CAAAA,MAAQ;IACRA,6BAAAA,CAAAA,CAAAA,MACS,CAAC;QACN,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBN,6BAAAA,CAAAA,CAAAA,MACrB,CAAC;IACN,QAAQI;IACR,WAAWJ,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,WAAWA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,OAAOA,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,OAAS;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAGiBA,6BAAAA,CAAAA,CAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,6BAAAA,CAAAA,CAAAA,OAAS,CAAC;IACjC,QAAQI;IAGR,WAAWJ,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,WAAWA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,OAAOA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAG3B,QAAQA,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,MAAQ;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAYP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIL,uBAAuBM,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACH;IACvC3E,IAAAA,sBAAAA,MAAAA,AAAAA,EACE0E,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOI,OAAOJ;AAChB;AAEO,MAAMK,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAO5G,OAAO,IAAI,CAAC2G,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAAC7G;YAClB,MAAMoG,QAAQQ,UAAU,KAAK,CAAC5G,IAAI;YAClC,IAAI,CAACmG,uBAAuBC,QAC1B,OAAO;YAIT,IAAIO,cACF,OAAOP,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMZ,kBAAkB,CAC7BsB,YACAC;IAEA,MAAMC,gBAAgBP,4BAA4BM;IAClD,MAAMzF,SAAS;QAAE,GAAGwF,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAa5F,MAAM,CAAC2F,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACT5F,MAAM,CAAC2F,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1B5F,MAAM,CAAC2F,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxB5F,MAAM,CAAC2F,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAO5F;AACT;AAEO,MAAM6F,kBAAkB,CAC7BL,YACAC;IAEA,MAAMC,gBAAgBP,4BAA4BM;IAClD,MAAMzF,SAAS;QAAE,GAAGwF,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAa5F,MAAM,CAAC2F,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvB5F,MAAM,CAAC2F,UAAU,GAAG;YAClB,CAACjB,oBAAoB,EAAE;YACvB,QAAQkB;QACV;IAEJ;IAEA,OAAO5F;AACT;AAUO,MAAM8F,mBAAmB,CAC9BC,UACAN;IAGA,IAAI,CAACA,WACH;IAIF,MAAMO,QAAQD,YAAY,CAAC;IAG3B,MAAME,eAAed,4BAA4BM;IAGjD,IAAIQ,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOR,UAAU,KAAK,CAACO;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMzH,OAAOsH,MAChB,IAAIC,aAAa,QAAQ,CAACvH,MAExByH,mBAAmB,CAACzH,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/CyH,mBAAmB,CAACzH,IAAI,GAAGsH,KAAK,CAACtH,IAAI;IAKzC,MAAM0H,YAAYX,UAAU,KAAK,CAACU;IAGlC,IAAK,MAAMR,aAAaO,kBACtBE,SAAS,CAACT,UAAU,GAAGO,iBAAiB,CAACP,UAAU;IAGrD,OAAOS;AACT"}
1	+ {"version":3,"file":"common.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/common.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n BaseElement,\n DeviceAction,\n ElementTreeNode,\n MidsceneYamlFlowItem,\n PlanningAction,\n Rect,\n Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nexport enum AIActionType {\n ASSERT = 0,\n INSPECT_ELEMENT = 1,\n EXTRACT_DATA = 2,\n PLAN = 3,\n DESCRIBE_ELEMENT = 4,\n TEXT = 5,\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\ntype AdaptBboxInput = number[] \| string[] \| string \| (number[] \| string[])[];\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n locate: PlanningLocateParam,\n width: number,\n height: number,\n rightLimit: number,\n bottomLimit: number,\n vlMode: TVlModeTypes \| undefined,\n) {\n // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n if ((locate as any).bbox_2d && !locate?.bbox) {\n locate.bbox = (locate as any).bbox_2d;\n // biome-ignore lint/performance/noDelete: <explanation>\n delete (locate as any).bbox_2d;\n }\n\n if (locate?.bbox) {\n locate.bbox = adaptBbox(\n locate.bbox,\n width,\n height,\n rightLimit,\n bottomLimit,\n vlMode,\n );\n }\n\n return locate;\n}\n\nexport function adaptQwen2_5Bbox(\n bbox: number[],\n): [number, number, number, number] {\n if (bbox.length < 2) {\n const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n throw new Error(msg);\n }\n\n const result: [number, number, number, number] = [\n Math.round(bbox[0]),\n Math.round(bbox[1]),\n typeof bbox[2] === 'number'\n ? Math.round(bbox[2])\n : Math.round(bbox[0] + defaultBboxSize),\n typeof bbox[3] === 'number'\n ? Math.round(bbox[3])\n : Math.round(bbox[1] + defaultBboxSize),\n ];\n return result;\n}\n\nexport function adaptDoubaoBbox(\n bbox: string[] \| number[] \| string,\n width: number,\n height: number,\n): [number, number, number, number] {\n assert(\n width > 0 && height > 0,\n 'width and height must be greater than 0 in doubao mode',\n );\n\n if (typeof bbox === 'string') {\n assert(\n /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n `invalid bbox data string for doubao-vision mode: ${bbox}`,\n );\n const splitted = bbox.split(' ');\n if (splitted.length === 4) {\n return [\n Math.round((Number(splitted[0]) * width) / 1000),\n Math.round((Number(splitted[1]) * height) / 1000),\n Math.round((Number(splitted[2]) * width) / 1000),\n Math.round((Number(splitted[3]) * height) / 1000),\n ];\n }\n throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n }\n\n let bboxList: number[] = [];\n if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n bbox.forEach((item) => {\n if (typeof item === 'string' && item.includes(',')) {\n const [x, y] = item.split(',');\n bboxList.push(Number(x.trim()), Number(y.trim()));\n } else if (typeof item === 'string' && item.includes(' ')) {\n const [x, y] = item.split(' ');\n bboxList.push(Number(x.trim()), Number(y.trim()));\n } else {\n bboxList.push(Number(item));\n }\n });\n } else {\n bboxList = bbox as any;\n }\n\n if (bboxList.length === 4 \|\| bboxList.length === 5) {\n return [\n Math.round((bboxList[0] * width) / 1000),\n Math.round((bboxList[1] * height) / 1000),\n Math.round((bboxList[2] * width) / 1000),\n Math.round((bboxList[3] * height) / 1000),\n ];\n }\n\n // treat the bbox as a center point\n if (\n bboxList.length === 6 \|\|\n bboxList.length === 2 \|\|\n bboxList.length === 3 \|\|\n bboxList.length === 7\n ) {\n return [\n Math.max(\n 0,\n Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n ),\n Math.max(\n 0,\n Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n ),\n Math.min(\n width,\n Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n ),\n Math.min(\n height,\n Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n ),\n ];\n }\n\n if (bbox.length === 8) {\n return [\n Math.round((bboxList[0] * width) / 1000),\n Math.round((bboxList[1] * height) / 1000),\n Math.round((bboxList[4] * width) / 1000),\n Math.round((bboxList[5] * height) / 1000),\n ];\n }\n\n const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n throw new Error(msg);\n}\n\nfunction normalizeBboxInput(\n bbox: AdaptBboxInput,\n): number[] \| string[] \| string {\n if (Array.isArray(bbox)) {\n if (Array.isArray(bbox[0])) {\n return bbox[0] as number[] \| string[];\n }\n return bbox as number[] \| string[];\n }\n return bbox as string;\n}\n\nexport function adaptBbox(\n bbox: AdaptBboxInput,\n width: number,\n height: number,\n rightLimit: number,\n bottomLimit: number,\n vlMode: TVlModeTypes \| undefined,\n): [number, number, number, number] {\n const normalizedBbox = normalizeBboxInput(bbox);\n\n let result: [number, number, number, number] = [0, 0, 0, 0];\n if (vlMode === 'doubao-vision' \|\| vlMode === 'vlm-ui-tars') {\n result = adaptDoubaoBbox(normalizedBbox, width, height);\n } else if (vlMode === 'gemini') {\n result = adaptGeminiBbox(normalizedBbox as number[], width, height);\n } else if (vlMode === 'qwen3-vl') {\n result = normalized01000(normalizedBbox as number[], width, height);\n } else {\n result = adaptQwen2_5Bbox(normalizedBbox as number[]);\n }\n\n result[2] = Math.min(result[2], rightLimit);\n result[3] = Math.min(result[3], bottomLimit);\n\n return result;\n}\n\n// x1, y1, x2, y2 -> 0-1000\nexport function normalized01000(\n bbox: number[],\n width: number,\n height: number,\n): [number, number, number, number] {\n return [\n Math.round((bbox[0] * width) / 1000),\n Math.round((bbox[1] * height) / 1000),\n Math.round((bbox[2] * width) / 1000),\n Math.round((bbox[3] * height) / 1000),\n ];\n}\n\n// y1, x1, y2, x2 -> 0-1000\nexport function adaptGeminiBbox(\n bbox: number[],\n width: number,\n height: number,\n): [number, number, number, number] {\n const left = Math.round((bbox[1] * width) / 1000);\n const top = Math.round((bbox[0] * height) / 1000);\n const right = Math.round((bbox[3] * width) / 1000);\n const bottom = Math.round((bbox[2] * height) / 1000);\n return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n bbox: number[],\n width: number,\n height: number,\n offsetX = 0,\n offsetY = 0,\n rightLimit = width,\n bottomLimit = height,\n vlMode?: TVlModeTypes \| undefined,\n): Rect {\n debugInspectUtils(\n 'adaptBboxToRect',\n bbox,\n width,\n height,\n 'offset',\n offsetX,\n offsetY,\n 'limit',\n rightLimit,\n bottomLimit,\n 'vlMode',\n vlMode,\n );\n const [left, top, right, bottom] = adaptBbox(\n bbox,\n width,\n height,\n rightLimit,\n bottomLimit,\n vlMode,\n );\n\n // Calculate initial rect dimensions\n const rectLeft = left;\n const rectTop = top;\n let rectWidth = right - left;\n let rectHeight = bottom - top;\n\n // Ensure the rect doesn't exceed image boundaries\n // If right edge exceeds width, adjust the width\n if (rectLeft + rectWidth > width) {\n rectWidth = width - rectLeft;\n }\n\n // If bottom edge exceeds height, adjust the height\n if (rectTop + rectHeight > height) {\n rectHeight = height - rectTop;\n }\n\n // Ensure minimum dimensions (width and height should be at least 1)\n rectWidth = Math.max(1, rectWidth);\n rectHeight = Math.max(1, rectHeight);\n\n const rect = {\n left: rectLeft + offsetX,\n top: rectTop + offsetY,\n width: rectWidth,\n height: rectHeight,\n };\n debugInspectUtils('adaptBboxToRect, result=', rect);\n\n return rect;\n}\n\nexport function mergeRects(rects: Rect[]) {\n const minLeft = Math.min(...rects.map((r) => r.left));\n const minTop = Math.min(...rects.map((r) => r.top));\n const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n return {\n left: minLeft,\n top: minTop,\n width: maxRight - minLeft,\n height: maxBottom - minTop,\n };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n rect: Rect,\n screenSize: Size,\n vlMode: TVlModeTypes \| undefined,\n) {\n let minEdgeSize = 500;\n if (vlMode === 'qwen3-vl') {\n minEdgeSize = 700;\n }\n const defaultPadding = 160;\n\n // Calculate padding needed to reach minimum edge size\n const paddingSizeHorizontal =\n rect.width < minEdgeSize\n ? Math.ceil((minEdgeSize - rect.width) / 2)\n : defaultPadding;\n const paddingSizeVertical =\n rect.height < minEdgeSize\n ? Math.ceil((minEdgeSize - rect.height) / 2)\n : defaultPadding;\n\n // Calculate new dimensions (ensure minimum edge size)\n let newWidth = Math.max(minEdgeSize, rect.width + paddingSizeHorizontal * 2);\n let newHeight = Math.max(minEdgeSize, rect.height + paddingSizeVertical * 2);\n\n // Calculate initial position with padding\n let newLeft = rect.left - paddingSizeHorizontal;\n let newTop = rect.top - paddingSizeVertical;\n\n // Ensure the rect doesn't exceed screen boundaries by adjusting position\n // If the rect goes beyond the right edge, shift it left\n if (newLeft + newWidth > screenSize.width) {\n newLeft = screenSize.width - newWidth;\n }\n\n // If the rect goes beyond the bottom edge, shift it up\n if (newTop + newHeight > screenSize.height) {\n newTop = screenSize.height - newHeight;\n }\n\n // Ensure the rect doesn't go beyond the left/top edges\n newLeft = Math.max(0, newLeft);\n newTop = Math.max(0, newTop);\n\n // If after position adjustment, the rect still exceeds screen boundaries,\n // clamp the dimensions to fit within screen\n if (newLeft + newWidth > screenSize.width) {\n newWidth = screenSize.width - newLeft;\n }\n if (newTop + newHeight > screenSize.height) {\n newHeight = screenSize.height - newTop;\n }\n\n rect.left = newLeft;\n rect.top = newTop;\n rect.width = newWidth;\n rect.height = newHeight;\n\n return rect;\n}\n\nexport async function markupImageForLLM(\n screenshotBase64: string,\n tree: ElementTreeNode<BaseElement>,\n size: Size,\n) {\n const elementsInfo = treeToList(tree);\n const elementsPositionInfoWithoutText = elementsInfo!.filter(\n (elementInfo) => {\n if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n return false;\n }\n return true;\n },\n );\n\n const imagePayload = await compositeElementInfoImg({\n inputImgBase64: screenshotBase64,\n elementsPositionInfo: elementsPositionInfoWithoutText,\n size,\n });\n return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n plans: PlanningAction[],\n actionSpace: DeviceAction<any>[],\n sleep?: number,\n): MidsceneYamlFlowItem[] {\n const flow: MidsceneYamlFlowItem[] = [];\n\n for (const plan of plans) {\n const verb = plan.type;\n\n const action = actionSpace.find((action) => action.name === verb);\n if (!action) {\n console.warn(\n `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n );\n continue;\n }\n\n const flowKey = action.interfaceAlias \|\| verb;\n const flowParam = action.paramSchema\n ? dumpActionParam(plan.param \|\| {}, action.paramSchema)\n : {};\n\n const flowItem: MidsceneYamlFlowItem = {\n [flowKey]: '',\n ...flowParam,\n };\n\n flow.push(flowItem);\n }\n\n if (sleep) {\n flow.push({\n sleep,\n });\n }\n\n return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n left: z.number(),\n top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n width: z.number(),\n height: z.number(),\n dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n z.object({\n zoom: z.number().optional(),\n }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n images: z\n .array(\n z.object({\n name: z.string(),\n url: z.string(),\n }),\n )\n .optional(),\n convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n z.string(),\n z\n .object({\n prompt: z.string(),\n })\n .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n .object({\n prompt: TUserPromptSchema,\n deepThink: z.boolean().optional(),\n cacheable: z.boolean().optional(),\n xpath: z.union([z.string(), z.boolean()]).optional(),\n })\n .passthrough();\n\n// Schema for locator field result (when AI returns locate results)\nconst MidsceneLocationResult = z\n .object({\n [locateFieldFlagName]: z.literal(true),\n prompt: TUserPromptSchema,\n\n // optional fields\n deepThink: z.boolean().optional(), // only available in vl model\n cacheable: z.boolean().optional(),\n xpath: z.boolean().optional(), // preset result for xpath\n\n // these two fields will only appear in the result\n center: z.tuple([z.number(), z.number()]),\n rect: RectSchema,\n })\n .passthrough();\n\n// Export the result type - this is used for runtime results that include center and rect\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\n\n// Export the input type - this is the inferred type from getMidsceneLocationSchema()\nexport type MidsceneLocationInputType = z.infer<typeof MidsceneLocationInput>;\n\n/*\n Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n /\nexport const getMidsceneLocationSchema = () => {\n return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n // Handle optional fields by getting the inner type\n let actualField = field;\n if (actualField._def?.typeName === 'ZodOptional') {\n actualField = actualField._def.innerType;\n }\n\n // Check if this is a ZodObject\n if (actualField._def?.typeName === 'ZodObject') {\n const shape = actualField._def.shape();\n\n // Method 1: Check for the location field flag (for result schema)\n if (locateFieldFlagName in shape) {\n return true;\n }\n\n // Method 2: Check if it's the input schema by checking for 'prompt' field\n // Input schema has 'prompt' as a required field\n if ('prompt' in shape && shape.prompt) {\n return true;\n }\n }\n\n return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n assert(\n ifMidsceneLocatorField(field),\n 'field is not a midscene locator field',\n );\n\n // If field is a string, return it directly\n if (typeof field === 'string') {\n return field;\n }\n\n // If field is an object with prompt property\n if (field && typeof field === 'object' && field.prompt) {\n // If prompt is a string, return it directly\n if (typeof field.prompt === 'string') {\n return field.prompt;\n }\n // If prompt is a TUserPrompt object, extract the prompt string\n if (typeof field.prompt === 'object' && field.prompt.prompt) {\n return field.prompt.prompt; // TODO: dump images if necessary\n }\n }\n\n // Fallback: try to convert to string\n return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n zodType?: z.ZodType<any>,\n requiredOnly?: boolean,\n): string[] => {\n if (!zodType) {\n return [];\n }\n\n // Check if this is a ZodObject by checking if it has a shape property\n const zodObject = zodType as any;\n if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n const keys = Object.keys(zodObject.shape);\n return keys.filter((key) => {\n const field = zodObject.shape[key];\n if (!ifMidsceneLocatorField(field)) {\n return false;\n }\n\n // If requiredOnly is true, filter out optional fields\n if (requiredOnly) {\n return field._def?.typeName !== 'ZodOptional';\n }\n\n return true;\n });\n }\n\n // For other ZodType instances, we can't extract field names\n return [];\n};\n\nexport const dumpActionParam = (\n jsonObject: Record<string, any>,\n zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n const locatorFields = findAllMidsceneLocatorField(zodSchema);\n const result = { ...jsonObject };\n\n for (const fieldName of locatorFields) {\n const fieldValue = result[fieldName];\n if (fieldValue) {\n // If it's already a string, keep it as is\n if (typeof fieldValue === 'string') {\n result[fieldName] = fieldValue;\n } else if (typeof fieldValue === 'object') {\n // Check if this field is actually a MidsceneLocationType object\n if (fieldValue.prompt) {\n // If prompt is a string, use it directly\n if (typeof fieldValue.prompt === 'string') {\n result[fieldName] = fieldValue.prompt;\n } else if (\n typeof fieldValue.prompt === 'object' &&\n fieldValue.prompt.prompt\n ) {\n // If prompt is a TUserPrompt object, extract the prompt string\n result[fieldName] = fieldValue.prompt.prompt;\n }\n }\n }\n }\n }\n\n return result;\n};\n\nexport const loadActionParam = (\n jsonObject: Record<string, any>,\n zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n const locatorFields = findAllMidsceneLocatorField(zodSchema);\n const result = { ...jsonObject };\n\n for (const fieldName of locatorFields) {\n const fieldValue = result[fieldName];\n if (fieldValue && typeof fieldValue === 'string') {\n result[fieldName] = {\n [locateFieldFlagName]: true,\n prompt: fieldValue,\n };\n }\n }\n\n return result;\n};\n\n/\n Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n \n Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n */\nexport const parseActionParam = (\n rawParam: Record<string, any> \| undefined,\n zodSchema?: z.ZodType<any>,\n): Record<string, any> \| undefined => {\n // If no schema is provided, return undefined (action takes no parameters)\n if (!zodSchema) {\n return undefined;\n }\n\n // Handle undefined or null rawParam by providing an empty object\n const param = rawParam ?? {};\n\n // Find all locate fields in the schema\n const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n // If there are no locate fields, just do normal validation\n if (locateFields.length === 0) {\n return zodSchema.parse(param);\n }\n\n // Extract locate field values to restore later\n const locateFieldValues: Record<string, any> = {};\n for (const fieldName of locateFields) {\n if (fieldName in param) {\n locateFieldValues[fieldName] = param[fieldName];\n }\n }\n\n // Build params for validation - skip locate fields and use dummy values\n const paramsForValidation: Record<string, any> = {};\n for (const key in param) {\n if (locateFields.includes(key)) {\n // Use dummy value to satisfy schema validation\n paramsForValidation[key] = { prompt: '_dummy_' };\n } else {\n paramsForValidation[key] = param[key];\n }\n }\n\n // Validate with dummy locate values\n const validated = zodSchema.parse(paramsForValidation);\n\n // Restore the actual locate field values (unvalidated, as per business requirement)\n for (const fieldName in locateFieldValues) {\n validated[fieldName] = locateFieldValues[fieldName];\n }\n\n return validated;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","AIActionType","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","rightLimit","bottomLimit","vlMode","adaptBbox","adaptQwen2_5Bbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","bboxList","Array","item","x","y","normalizeBboxInput","normalizedBbox","adaptGeminiBbox","normalized01000","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rectLeft","rectTop","rectWidth","rectHeight","rect","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","size","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","console","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","jsonObject","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam","parseActionParam","rawParam","param","locateFields","locateFieldValues","paramsForValidation","validated"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACiBO,IAAKI,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;;WAAZA;;AASZ,MAAMC,kBAAkB;AACxB,MAAMC,oBAAoBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAI5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAGhC,IAAKL,OAAe,OAAO,IAAI,CAACA,QAAQ,MAAM;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAQ,MACVA,OAAO,IAAI,GAAGM,UACZN,OAAO,IAAI,EACXC,OACAC,QACAC,YACAC,aACAC;IAIJ,OAAOL;AACT;AAEO,SAASO,iBACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;QACN,YAAnB,OAAOY,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;KAC1B;IACD,OAAOgB;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCP,KAAa,EACbC,MAAc;IAEda,IAAAA,sBAAAA,MAAAA,AAAAA,EACEd,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOM,MAAmB;QAC5BO,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;YAC5CW,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;SAC7C;QAEH,MAAM,IAAIS,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,WAAqB,EAAE;IAC3B,IAAIC,MAAM,OAAO,CAACX,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACD,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACD,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEJ,SAAS,IAAI,CAACD,OAAOG;IAEzB;SAEAF,WAAWV;IAGb,IAAIU,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLL,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;QACpCW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;KACrC;IAIH,IACEgB,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLL,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU,QAAQN,kBAAkB;QAEhEiB,KAAK,GAAG,CACNZ,OACAY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACNX,QACAW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIY,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;QACpCW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;KACrC;IAGH,MAAMO,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEA,SAASc,mBACPf,IAAoB;IAEpB,IAAIW,MAAM,OAAO,CAACX,OAChB;QAAA,IAAIW,MAAM,OAAO,CAACX,IAAI,CAAC,EAAE,GACvB,OAAOA,IAAI,CAAC,EAAE;IAChB;IAGF,OAAOA;AACT;AAEO,SAASF,UACdE,IAAoB,EACpBP,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAEhC,MAAMmB,iBAAiBD,mBAAmBf;IAE1C,IAAII,SAA2C;QAAC;QAAG;QAAG;QAAG;KAAE;IAEzDA,SADEP,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,SACvBS,gBAAgBU,gBAAgBvB,OAAOC,UACvCG,AAAW,aAAXA,SACAoB,gBAAgBD,gBAA4BvB,OAAOC,UACnDG,AAAW,eAAXA,SACAqB,gBAAgBF,gBAA4BvB,OAAOC,UAEnDK,iBAAiBiB;IAG5BZ,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAET;IAChCS,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAER;IAEhC,OAAOQ;AACT;AAGO,SAASc,gBACdlB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,OAAO;QACLW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;QAChCW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;KACjC;AACH;AAGO,SAASuB,gBACdjB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,MAAMyB,OAAOd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC5C,MAAM2B,MAAMf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC5C,MAAM2B,QAAQhB,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC7C,MAAM6B,SAASjB,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC/C,OAAO;QAACyB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdvB,IAAc,EACdP,KAAa,EACbC,MAAc,EACd8B,UAAU,CAAC,EACXC,UAAU,CAAC,EACX9B,aAAaF,KAAK,EAClBG,cAAcF,MAAM,EACpBG,MAAiC;IAEjCR,kBACE,mBACAW,MACAP,OACAC,QACA,UACA8B,SACAC,SACA,SACA9B,YACAC,aACA,UACAC;IAEF,MAAM,CAACsB,MAAMC,KAAKC,OAAOC,OAAO,GAAGxB,UACjCE,MACAP,OACAC,QACAC,YACAC,aACAC;IAIF,MAAM6B,WAAWP;IACjB,MAAMQ,UAAUP;IAChB,IAAIQ,YAAYP,QAAQF;IACxB,IAAIU,aAAaP,SAASF;IAI1B,IAAIM,WAAWE,YAAYnC,OACzBmC,YAAYnC,QAAQiC;IAItB,IAAIC,UAAUE,aAAanC,QACzBmC,aAAanC,SAASiC;IAIxBC,YAAYvB,KAAK,GAAG,CAAC,GAAGuB;IACxBC,aAAaxB,KAAK,GAAG,CAAC,GAAGwB;IAEzB,MAAMC,OAAO;QACX,MAAMJ,WAAWF;QACjB,KAAKG,UAAUF;QACf,OAAOG;QACP,QAAQC;IACV;IACAxC,kBAAkB,4BAA4ByC;IAE9C,OAAOA;AACT;AAEO,SAASC,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdR,IAAU,EACVS,UAAgB,EAChB1C,MAAgC;IAEhC,IAAI2C,cAAc;IAClB,IAAI3C,AAAW,eAAXA,QACF2C,cAAc;IAEhB,MAAMC,iBAAiB;IAGvB,MAAMC,wBACJZ,KAAK,KAAK,GAAGU,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAcV,KAAK,KAAI,IAAK,KACvCW;IACN,MAAME,sBACJb,KAAK,MAAM,GAAGU,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAcV,KAAK,MAAK,IAAK,KACxCW;IAGN,IAAIG,WAAWvC,KAAK,GAAG,CAACmC,aAAaV,KAAK,KAAK,GAAGY,AAAwB,IAAxBA;IAClD,IAAIG,YAAYxC,KAAK,GAAG,CAACmC,aAAaV,KAAK,MAAM,GAAGa,AAAsB,IAAtBA;IAGpD,IAAIG,UAAUhB,KAAK,IAAI,GAAGY;IAC1B,IAAIK,SAASjB,KAAK,GAAG,GAAGa;IAIxB,IAAIG,UAAUF,WAAWL,WAAW,KAAK,EACvCO,UAAUP,WAAW,KAAK,GAAGK;IAI/B,IAAIG,SAASF,YAAYN,WAAW,MAAM,EACxCQ,SAASR,WAAW,MAAM,GAAGM;IAI/BC,UAAUzC,KAAK,GAAG,CAAC,GAAGyC;IACtBC,SAAS1C,KAAK,GAAG,CAAC,GAAG0C;IAIrB,IAAID,UAAUF,WAAWL,WAAW,KAAK,EACvCK,WAAWL,WAAW,KAAK,GAAGO;IAEhC,IAAIC,SAASF,YAAYN,WAAW,MAAM,EACxCM,YAAYN,WAAW,MAAM,GAAGQ;IAGlCjB,KAAK,IAAI,GAAGgB;IACZhB,KAAK,GAAG,GAAGiB;IACXjB,KAAK,KAAK,GAAGc;IACbd,KAAK,MAAM,GAAGe;IAEd,OAAOf;AACT;AAEO,eAAekB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCC,IAAU;IAEV,MAAMC,eAAeC,AAAAA,IAAAA,0BAAAA,UAAAA,AAAAA,EAAWH;IAChC,MAAMI,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,0BAAAA,QAAAA,CAAAA,IAAa,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;QACjD,gBAAgBT;QAChB,sBAAsBK;QACtBH;IACF;IACA,OAAOM;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEF,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMG,UAAUF,OAAO,cAAc,IAAID;QACzC,MAAMI,YAAYH,OAAO,WAAW,GAChCI,gBAAgBN,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMK,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAN,KAAK,IAAI,CAACQ;IACZ;IAEA,IAAIT,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMS,cAAcC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClC,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;IACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;AACf;AAEO,MAAMC,aAAaD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjC,OAAOA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;IAChB,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,QAAQA,6BAAAA,CAAAA,CAAAA,KACA,CACJA,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;QACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;QACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,IAED,QAAQ;IACX,yBAAyBA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;IACvCA,6BAAAA,CAAAA,CAAAA,MAAQ;IACRA,6BAAAA,CAAAA,CAAAA,MACS,CAAC;QACN,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBN,6BAAAA,CAAAA,CAAAA,MACrB,CAAC;IACN,QAAQI;IACR,WAAWJ,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,WAAWA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,OAAOA,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,OAAS;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAGiBA,6BAAAA,CAAAA,CAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,6BAAAA,CAAAA,CAAAA,OAAS,CAAC;IACjC,QAAQI;IAGR,WAAWJ,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,WAAWA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,OAAOA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAG3B,QAAQA,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,MAAQ;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAYP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIL,uBAAuBM,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACH;IACvC3E,IAAAA,sBAAAA,MAAAA,AAAAA,EACE0E,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOI,OAAOJ;AAChB;AAEO,MAAMK,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAO5G,OAAO,IAAI,CAAC2G,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAAC7G;YAClB,MAAMoG,QAAQQ,UAAU,KAAK,CAAC5G,IAAI;YAClC,IAAI,CAACmG,uBAAuBC,QAC1B,OAAO;YAIT,IAAIO,cACF,OAAOP,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMZ,kBAAkB,CAC7BsB,YACAC;IAEA,MAAMC,gBAAgBP,4BAA4BM;IAClD,MAAMzF,SAAS;QAAE,GAAGwF,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAa5F,MAAM,CAAC2F,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACT5F,MAAM,CAAC2F,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1B5F,MAAM,CAAC2F,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxB5F,MAAM,CAAC2F,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAO5F;AACT;AAEO,MAAM6F,kBAAkB,CAC7BL,YACAC;IAEA,MAAMC,gBAAgBP,4BAA4BM;IAClD,MAAMzF,SAAS;QAAE,GAAGwF,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAa5F,MAAM,CAAC2F,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvB5F,MAAM,CAAC2F,UAAU,GAAG;YAClB,CAACjB,oBAAoB,EAAE;YACvB,QAAQkB;QACV;IAEJ;IAEA,OAAO5F;AACT;AAUO,MAAM8F,mBAAmB,CAC9BC,UACAN;IAGA,IAAI,CAACA,WACH;IAIF,MAAMO,QAAQD,YAAY,CAAC;IAG3B,MAAME,eAAed,4BAA4BM;IAGjD,IAAIQ,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOR,UAAU,KAAK,CAACO;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMzH,OAAOsH,MAChB,IAAIC,aAAa,QAAQ,CAACvH,MAExByH,mBAAmB,CAACzH,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/CyH,mBAAmB,CAACzH,IAAI,GAAGsH,KAAK,CAACtH,IAAI;IAKzC,MAAM0H,YAAYX,UAAU,KAAK,CAACU;IAGlC,IAAK,MAAMR,aAAaO,kBACtBE,SAAS,CAACT,UAAU,GAAGO,iBAAiB,CAACP,UAAU;IAGrD,OAAOS;AACT"}

package/dist/lib/index.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/index.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { z } from 'zod';\nimport Service from './service/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n plan,\n describeUserPage,\n AiLocateElement,\n getMidsceneLocationSchema,\n type MidsceneLocationResultType,\n PointSchema,\n SizeSchema,\n RectSchema,\n TMultimodalPromptSchema,\n TUserPromptSchema,\n type TMultimodalPrompt,\n type TUserPrompt,\n} from './ai-model/index';\n\nexport {\n MIDSCENE_MODEL_NAME,\n type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport { ServiceError } from './types';\n\nexport { z };\n\nexport default Service;\nexport { TaskRunner, Service, getVersion };\n\nexport type {\n MidsceneYamlScript,\n MidsceneYamlTask,\n MidsceneYamlFlowItem,\n MidsceneYamlConfigResult,\n MidsceneYamlConfig,\n MidsceneYamlScriptWebEnv,\n MidsceneYamlScriptAndroidEnv,\n MidsceneYamlScriptIOSEnv,\n MidsceneYamlScriptEnv,\n LocateOption,\n DetailedLocateParam,\n} from './yaml';\n\nexport { Agent, type AgentOpt, createAgent } from './agent';\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","Service"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACwBA,YAAeI"}
1	+ {"version":3,"file":"index.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/index.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { z } from 'zod';\nimport Service from './service/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n plan,\n describeUserPage,\n AiLocateElement,\n getMidsceneLocationSchema,\n type MidsceneLocationResultType,\n PointSchema,\n SizeSchema,\n RectSchema,\n TMultimodalPromptSchema,\n TUserPromptSchema,\n type TMultimodalPrompt,\n type TUserPrompt,\n} from './ai-model/index';\n\nexport {\n MIDSCENE_MODEL_NAME,\n type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport { ServiceError } from './types';\n\nexport { z };\n\nexport default Service;\nexport { TaskRunner, Service, getVersion };\n\nexport type {\n MidsceneYamlScript,\n MidsceneYamlTask,\n MidsceneYamlFlowItem,\n MidsceneYamlConfigResult,\n MidsceneYamlConfig,\n MidsceneYamlScriptWebEnv,\n MidsceneYamlScriptAndroidEnv,\n MidsceneYamlScriptIOSEnv,\n MidsceneYamlScriptEnv,\n LocateOption,\n DetailedLocateParam,\n} from './yaml';\n\nexport { Agent, type AgentOpt, type AiActOptions, createAgent } from './agent';\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","Service"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACwBA,YAAeI"}