@aiscene/core 8.0.1 → 8.0.3

package/dist/lib/ai-model/prompt/extraction.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { AIDataExtractionResponse } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\nimport { safeParseJson } from '../service-caller/index';\nimport { extractXMLTag } from './util';\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = safeParseJson(dataJsonStr, undefined) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = safeParseJson(errorsStr, undefined);\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract() {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\nThe user will give you a screenshot, the contents of it (optional), and some data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nIMPORTANT: For Boolean type queries, you MUST base your answer ONLY on what is directly visible in the screenshot. Do NOT speculate, infer, or assume anything that is not explicitly shown in the image. If the information is not clearly visible or cannot be determined from the screenshot, you MUST return false. Never fabricate or imagine content that is not present in the image.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\nIf the user provides multiple reference images, please carefully review the reference images with the screenshot and provide the correct answer for <DATA_DEMAND>.\n\n\nReturn in the following XML format:\n<thought>the thinking process of the extraction, less than 300 words. Use ${preferredLanguage} in this field.</thought>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","safeParseJson","undefined","e","errors","parsedErrors","Array","systemPromptToExtract","preferredLanguage","getPreferredLanguage","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACGO,SAASI,2BACdC,SAAiB;IAEjB,MAAMC,UAAUC,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IACzC,MAAMG,cAAcD,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IAC7C,MAAMI,YAAYF,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,AAAAA,IAAAA,yBAAAA,aAAAA,AAAAA,EAAcJ,aAAaK;IACpC,EAAE,OAAOC,GAAG;QACV,MAAM,IAAIJ,MAAM,CAAC,2BAA2B,EAAEI,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIN,WACF,IAAI;QACF,MAAMO,eAAeJ,AAAAA,IAAAA,yBAAAA,aAAAA,AAAAA,EAAcH,WAAWI;QAC9C,IAAII,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIR,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAII,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG;IACd,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;0EAegE,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqE9F,CAAC;AACD;AAEO,MAAME,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}
+{"version":3,"file":"ai-model/prompt/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { AIDataExtractionResponse } from '@/types';\nimport { getPreferredLanguage } from '@aiscene/shared/env';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\nimport { safeParseJson } from '../service-caller/index';\nimport { extractXMLTag } from './util';\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = safeParseJson(dataJsonStr, undefined) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = safeParseJson(errorsStr, undefined);\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract() {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\nThe user will give you a screenshot, the contents of it (optional), and some data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nIMPORTANT: For Boolean type queries, you MUST base your answer ONLY on what is directly visible in the screenshot. Do NOT speculate, infer, or assume anything that is not explicitly shown in the image. If the information is not clearly visible or cannot be determined from the screenshot, you MUST return false. Never fabricate or imagine content that is not present in the image.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\nIf the user provides multiple reference images, please carefully review the reference images with the screenshot and provide the correct answer for <DATA_DEMAND>.\n\n\nReturn in the following XML format:\n<thought>the thinking process of the extraction, less than 300 words. Use ${preferredLanguage} in this field.</thought>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","safeParseJson","undefined","e","errors","parsedErrors","Array","systemPromptToExtract","preferredLanguage","getPreferredLanguage","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACGO,SAASI,2BACdC,SAAiB;IAEjB,MAAMC,UAAUC,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IACzC,MAAMG,cAAcD,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IAC7C,MAAMI,YAAYF,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,AAAAA,IAAAA,yBAAAA,aAAAA,AAAAA,EAAcJ,aAAaK;IACpC,EAAE,OAAOC,GAAG;QACV,MAAM,IAAIJ,MAAM,CAAC,2BAA2B,EAAEI,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIN,WACF,IAAI;QACF,MAAMO,eAAeJ,AAAAA,IAAAA,yBAAAA,aAAAA,AAAAA,EAAcH,WAAWI;QAC9C,IAAII,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIR,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAII,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG;IACd,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;0EAegE,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqE9F,CAAC;AACD;AAEO,MAAME,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}

package/dist/lib/ai-model/prompt/llm-locator.js

@@ -27,7 +27,7 @@ __webpack_require__.d(__webpack_exports__, {
     systemPromptToLocateElement: ()=>systemPromptToLocateElement,
     findElementPrompt: ()=>findElementPrompt
 });
-const env_namespaceObject = require("@midscene/shared/env");
+const env_namespaceObject = require("@aiscene/shared/env");
 const external_common_js_namespaceObject = require("./common.js");
 function systemPromptToLocateElement(modelFamily) {
     const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();

package/dist/lib/ai-model/prompt/llm-locator.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/llm-locator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-locator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { bboxDescription } from './common';\nexport function systemPromptToLocateElement(\n  modelFamily: TModelFamily | undefined,\n) {\n  const preferredLanguage = getPreferredLanguage();\n  const bboxComment = bboxDescription(modelFamily);\n  return `\n## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Identify elements in screenshots that match the user's description.\n- Provide the coordinates of the element that matches the user's description.\n\n## Important Notes for Locating Elements:\n- When the user describes an element that contains text (such as buttons, input fields, dropdown options, radio buttons, etc.), you should locate ONLY the text region of that element, not the entire element boundary.\n- For example: If an input field is large (both wide and tall) with a placeholder text \"Please enter your comment\", you should locate only the area where the placeholder text appears, not the entire input field.\n- This principle applies to all text-containing elements: focus on the visible text region rather than the full element container.\n\n## Output Format:\n\\`\\`\\`json\n{\n  \"bbox\": [number, number, number, number],  // ${bboxComment}\n  \"errors\"?: string[]\n}\n\\`\\`\\`\n\nFields:\n* \\`bbox\\` is the bounding box of the element that matches the user's description\n* \\`errors\\` is an optional array of error messages (if any)\n\nFor example, when an element is found:\n\\`\\`\\`json\n{\n  \"bbox\": [100, 100, 200, 200],\n  \"errors\": []\n}\n\\`\\`\\`\n\nWhen no element is found:\n\\`\\`\\`json\n{\n  \"bbox\": [],\n  \"errors\": [\"I can see ..., but {some element} is not found. Use ${preferredLanguage}.\"]\n}\n\\`\\`\\`\n`;\n}\n\nexport const findElementPrompt = (targetElementDescription: string) =>\n  `Find: ${targetElementDescription}`;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","systemPromptToLocateElement","modelFamily","preferredLanguage","getPreferredLanguage","bboxComment","bboxDescription","findElementPrompt","targetElementDescription"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACHO,SAASI,4BACdC,WAAqC;IAErC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAC1B,MAAMC,cAAcC,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EAAgBJ;IACpC,OAAO,CAAC;;;;;;;;;;;;;;;;gDAgBsC,EAAEG,YAAY;;;;;;;;;;;;;;;;;;;;;kEAqBI,EAAEF,kBAAkB;;;AAGtF,CAAC;AACD;AAEO,MAAMI,oBAAoB,CAACC,2BAChC,CAAC,MAAM,EAAEA,0BAA0B"}
+{"version":3,"file":"ai-model/prompt/llm-locator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-locator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@aiscene/shared/env';\nimport { getPreferredLanguage } from '@aiscene/shared/env';\nimport { bboxDescription } from './common';\nexport function systemPromptToLocateElement(\n  modelFamily: TModelFamily | undefined,\n) {\n  const preferredLanguage = getPreferredLanguage();\n  const bboxComment = bboxDescription(modelFamily);\n  return `\n## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Identify elements in screenshots that match the user's description.\n- Provide the coordinates of the element that matches the user's description.\n\n## Important Notes for Locating Elements:\n- When the user describes an element that contains text (such as buttons, input fields, dropdown options, radio buttons, etc.), you should locate ONLY the text region of that element, not the entire element boundary.\n- For example: If an input field is large (both wide and tall) with a placeholder text \"Please enter your comment\", you should locate only the area where the placeholder text appears, not the entire input field.\n- This principle applies to all text-containing elements: focus on the visible text region rather than the full element container.\n\n## Output Format:\n\\`\\`\\`json\n{\n  \"bbox\": [number, number, number, number],  // ${bboxComment}\n  \"errors\"?: string[]\n}\n\\`\\`\\`\n\nFields:\n* \\`bbox\\` is the bounding box of the element that matches the user's description\n* \\`errors\\` is an optional array of error messages (if any)\n\nFor example, when an element is found:\n\\`\\`\\`json\n{\n  \"bbox\": [100, 100, 200, 200],\n  \"errors\": []\n}\n\\`\\`\\`\n\nWhen no element is found:\n\\`\\`\\`json\n{\n  \"bbox\": [],\n  \"errors\": [\"I can see ..., but {some element} is not found. Use ${preferredLanguage}.\"]\n}\n\\`\\`\\`\n`;\n}\n\nexport const findElementPrompt = (targetElementDescription: string) =>\n  `Find: ${targetElementDescription}`;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","systemPromptToLocateElement","modelFamily","preferredLanguage","getPreferredLanguage","bboxComment","bboxDescription","findElementPrompt","targetElementDescription"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACHO,SAASI,4BACdC,WAAqC;IAErC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAC1B,MAAMC,cAAcC,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EAAgBJ;IACpC,OAAO,CAAC;;;;;;;;;;;;;;;;gDAgBsC,EAAEG,YAAY;;;;;;;;;;;;;;;;;;;;;kEAqBI,EAAEF,kBAAkB;;;AAGtF,CAAC;AACD;AAEO,MAAMI,oBAAoB,CAACC,2BAChC,CAAC,MAAM,EAAEA,0BAA0B"}

package/dist/lib/ai-model/prompt/llm-planning.js

@@ -28,8 +28,8 @@ __webpack_require__.d(__webpack_exports__, {
     systemPromptToTaskPlanning: ()=>systemPromptToTaskPlanning
 });
 const external_common_js_namespaceObject = require("../../common.js");
-const env_namespaceObject = require("@midscene/shared/env");
-const zod_schema_utils_namespaceObject = require("@midscene/shared/zod-schema-utils");
+const env_namespaceObject = require("@aiscene/shared/env");
+const zod_schema_utils_namespaceObject = require("@aiscene/shared/zod-schema-utils");
 const external_common_js_namespaceObject_1 = require("./common.js");
 const vlLocateParam = (modelFamily)=>{
     if (modelFamily) return `{bbox: [number, number, number, number], prompt: string } // ${(0, external_common_js_namespaceObject_1.bboxDescription)(modelFamily)}`;

package/dist/lib/ai-model/prompt/llm-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/types';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport {\n  getZodDescription,\n  getZodTypeName,\n} from '@midscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { bboxDescription } from './common';\n\nconst vlLocateParam = (modelFamily: TModelFamily | undefined) => {\n  if (modelFamily) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(modelFamily)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\n/**\n * Find ZodDefault in the wrapper chain and return its default value\n */\nconst findDefaultValue = (field: unknown): any | undefined => {\n  let current = field;\n  const visited = new Set<unknown>();\n\n  while (current && !visited.has(current)) {\n    visited.add(current);\n    const currentWithDef = current as {\n      _def?: {\n        typeName?: string;\n        defaultValue?: () => any;\n        innerType?: unknown;\n      };\n    };\n\n    if (!currentWithDef._def?.typeName) break;\n\n    if (currentWithDef._def.typeName === 'ZodDefault') {\n      return currentWithDef._def.defaultValue?.();\n    }\n\n    // Continue unwrapping if it's a wrapper type\n    if (\n      currentWithDef._def.typeName === 'ZodOptional' ||\n      currentWithDef._def.typeName === 'ZodNullable'\n    ) {\n      current = currentWithDef._def.innerType;\n    } else {\n      break;\n    }\n  }\n\n  return undefined;\n};\n\n/**\n * Inject bbox into locate fields of a sample object.\n * Walks the sample and for any locate field (identified by paramSchema),\n * adds a fake bbox array when includeBbox is true.\n */\nconst SAMPLE_BBOXES: [number, number, number, number][] = [\n  [50, 100, 200, 200],\n  [300, 400, 500, 500],\n  [600, 100, 800, 250],\n  [50, 600, 250, 750],\n];\n\nconst injectBboxIntoSample = (\n  sample: Record<string, any>,\n  locateFields: string[],\n  includeBbox: boolean,\n): Record<string, any> => {\n  if (!includeBbox) return sample;\n  const result = { ...sample };\n  let bboxIndex = 0;\n  for (const field of locateFields) {\n    if (\n      result[field] &&\n      typeof result[field] === 'object' &&\n      result[field].prompt\n    ) {\n      result[field] = {\n        ...result[field],\n        bbox: SAMPLE_BBOXES[bboxIndex % SAMPLE_BBOXES.length],\n      };\n      bboxIndex++;\n    }\n  }\n  return result;\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n  includeBbox = false,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  // Add the action type field\n  fields.push(`- type: \"${action.name}\"`);\n\n  // Handle paramSchema if it exists\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    // Check if paramSchema is a ZodObject with shape\n    const schema = action.paramSchema as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      // Original logic for ZodObject schemas\n      const shape = schema.shape;\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          // Check if field is optional\n          const isOptional =\n            typeof (field as { isOptional?: () => boolean }).isOptional ===\n              'function' &&\n            (field as { isOptional: () => boolean }).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          // Get the type name using extracted helper\n          const typeName = getZodTypeName(field, locatorSchemaTypeDescription);\n\n          // Get description using extracted helper\n          const description = getZodDescription(field as z.ZodTypeAny);\n\n          // Check if field has a default value by searching the wrapper chain\n          const defaultValue = findDefaultValue(field);\n          const hasDefault = defaultValue !== undefined;\n\n          // Build param line for this field\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          const comments: string[] = [];\n          if (description) {\n            comments.push(description);\n          }\n          if (hasDefault) {\n            const defaultStr =\n              typeof defaultValue === 'string'\n                ? `\"${defaultValue}\"`\n                : JSON.stringify(defaultValue);\n            comments.push(`default: ${defaultStr}`);\n          }\n          if (comments.length > 0) {\n            paramLine += ` // ${comments.join(', ')}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      // Add the param section to fields if there are paramLines\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      // Handle non-object schemas (string, number, etc.)\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(schema as z.ZodTypeAny);\n\n      // For simple types, indicate that param should be the direct value, not an object\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  // Render sample if provided, using the same XML tag format as the real output\n  if (action.sample && typeof action.sample === 'object') {\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n    const sampleWithBbox = injectBboxIntoSample(\n      action.sample,\n      locateFields,\n      includeBbox,\n    );\n    const sampleStr = `- sample:\\n${tab}${tab}<action-type>${action.name}</action-type>\\n${tab}${tab}<action-param-json>\\n${tab}${tab}${JSON.stringify(sampleWithBbox, null, 2).replace(/\\n/g, `\\n${tab}${tab}`)}\\n${tab}${tab}</action-param-json>`;\n    fields.push(sampleStr);\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  modelFamily,\n  includeBbox,\n  includeThought,\n  includeSubGoals,\n}: {\n  actionSpace: DeviceAction<any>[];\n  modelFamily: TModelFamily | undefined;\n  includeBbox: boolean;\n  includeThought?: boolean;\n  includeSubGoals?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n\n  // Validate parameters: if includeBbox is true, modelFamily must be defined\n  if (includeBbox && !modelFamily) {\n    throw new Error(\n      'modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? modelFamily : undefined),\n      includeBbox,\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const shouldIncludeThought = includeThought ?? true;\n  const shouldIncludeSubGoals = includeSubGoals ?? false;\n\n  // Generate locate object examples based on includeBbox\n  const locateExample1 = includeBbox\n    ? `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\",\n    \"bbox\": [345, 442, 458, 483]\n  }`\n    : `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\"\n  }`;\n\n  // Locate examples for multi-turn conversation\n  const locateNameField = includeBbox\n    ? `{\n    \"prompt\": \"Name input field in the registration form\",\n    \"bbox\": [120, 180, 380, 210]\n  }`\n    : `{\n    \"prompt\": \"Name input field in the registration form\"\n  }`;\n\n  const locateEmailField = includeBbox\n    ? `{\n    \"prompt\": \"Email input field in the registration form\",\n    \"bbox\": [120, 240, 380, 270]\n  }`\n    : `{\n    \"prompt\": \"Email input field in the registration form\"\n  }`;\n\n  const thoughtTag = (content: string) =>\n    shouldIncludeThought ? `<thought>${content}</thought>\\n` : '';\n\n  // Sub-goals related content - only included when shouldIncludeSubGoals is true\n  const step1Title = shouldIncludeSubGoals\n    ? '## Step 1: Observe and Plan (related tags: <thought>, <update-plan-content>, <mark-sub-goal-done>)'\n    : '## Step 1: Observe (related tags: <thought>)';\n\n  const step1Description = shouldIncludeSubGoals\n    ? \"First, observe the current screenshot and previous logs, then break down the user's instruction into multiple high-level sub-goals. Update the status of sub-goals based on what you see in the current screenshot.\"\n    : 'First, observe the current screenshot and previous logs to understand the current state.';\n\n  const explicitInstructionRule = `CRITICAL - Following Explicit Instructions: When the user gives you specific operation steps (not high-level goals), you MUST execute ONLY those exact steps - nothing more, nothing less. Do NOT add extra actions even if they seem logical. For example: \"fill out the form\" means only fill fields, do NOT submit; \"click the button\" means only click, do NOT wait for page load or verify results; \"type 'hello'\" means only type, do NOT press Enter.`;\n\n  const thoughtTagDescription = shouldIncludeSubGoals\n    ? `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`\n    : `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the current state based on the screenshot? What should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`;\n\n  const subGoalTags = shouldIncludeSubGoals\n    ? `\n\n* <update-plan-content> tag\n\nUse this structure to give or update your plan:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished|pending\">sub goal description</sub-goal>\n  <sub-goal index=\"2\" status=\"finished|pending\">sub goal description</sub-goal>\n  ...\n</update-plan-content>\n\n* <mark-sub-goal-done> tag\n\nUse this structure to mark a sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nIMPORTANT: You MUST only mark a sub-goal as \"finished\" AFTER you have confirmed the task is actually completed by observing the result in the screenshot. Do NOT mark a sub-goal as done just because you expect the next action will complete it. Wait until you see visual confirmation in the screenshot that the sub-goal has been achieved.\n\n* Note\n\nDuring execution, you can call <update-plan-content> at any time to update the plan based on the latest screenshot and completed sub-goals.\n\n### Example\n\nIf the user wants to \"log in to a system using username and password, complete all to-do items, and submit a registration form\", you can break it down into the following sub-goals:\n\n<thought>...</thought>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Log in to the system</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Complete all to-do items</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Submit the registration form</sub-goal>\n</update-plan-content>\n\nAfter logging in and seeing the to-do items, you can mark the sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nAt this point, the status of all sub-goals is:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished\" />\n  <sub-goal index=\"2\" status=\"pending\" />\n  <sub-goal index=\"3\" status=\"pending\" />\n</update-plan-content>\n\nAfter some time, when the last sub-goal is also completed, you can mark it as done as well:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>`\n    : '';\n\n  // Step numbering adjusts based on whether sub-goals are included\n  // When includeSubGoals=false, memory step is skipped\n  const memoryStepNumber = 2; // Only used when shouldIncludeSubGoals is true\n  const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;\n  const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;\n\n  return `\nTarget: You are an expert to manipulate the UI to accomplish the user's instruction. User will give you an instruction, some screenshots, background knowledge and previous logs indicating what have been done. Your task is to accomplish the instruction by thinking through the path to complete the task and give the next action to execute.\n\n${step1Title}\n\n${step1Description}\n\n* <thought> tag (REQUIRED)\n\n${thoughtTagDescription}\n${subGoalTags}\n${\n  shouldIncludeSubGoals\n    ? `\n## Step ${memoryStepNumber}: Memory Data from Current Screenshot (related tags: <memory>)\n\nWhile observing the current screenshot, if you notice any information that might be needed in follow-up actions, record it here. The current screenshot will NOT be available in subsequent steps, so this memory is your only way to preserve essential information. Examples: extracted data, element states, content that needs to be referenced.\n\nDon't use this tag if no information needs to be preserved.\n`\n    : ''\n}\n## Step ${checkGoalStepNumber}: ${shouldIncludeSubGoals ? 'Check if Goal is Accomplished' : 'Check if the Instruction is Fulfilled'} (related tags: <complete>)\n\n${shouldIncludeSubGoals ? 'Based on the current screenshot and the status of all sub-goals, determine' : 'Determine'} if the entire task is completed.\n\n### CRITICAL: The User's Instruction is the Supreme Authority\n\nThe user's instruction defines the EXACT scope of what you must accomplish. You MUST follow it precisely - nothing more, nothing less. Violating this rule may cause severe consequences such as data loss, unintended operations, or system failures.\n\n**Explicit instructions vs. High-level goals:**\n- If the user gives you **explicit operation steps** (e.g., \"click X\", \"type Y\", \"fill out the form\"), treat them as exact commands. Execute ONLY those steps, nothing more.\n- If the user gives you a **high-level goal** (e.g., \"log in to the system\", \"complete the purchase\"), you may determine the necessary steps to achieve it.\n\n**What \"${shouldIncludeSubGoals ? 'goal accomplished' : 'instruction fulfilled'}\" means:**\n- The ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} when you have done EXACTLY what the user asked - no extra steps, no assumptions.\n- Do NOT perform any action beyond the explicit instruction, even if it seems logical or helpful.\n\n**Examples - Explicit instructions (execute exactly, no extra steps):**\n- \"fill out the form\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when all fields are filled. Do NOT submit the form.\n- \"click the login button\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} once clicked. Do NOT wait for page load or verify login success.\n- \"type 'hello' in the search box\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when 'hello' is typed. Do NOT press Enter or trigger search.\n- \"select the first item\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when selected. Do NOT proceed to checkout.\n\n**Special case - Assertion instructions:**\n- If the user's instruction includes an assertion (e.g., \"verify that...\", \"check that...\", \"assert...\"), and you observe from the screenshot that the assertion condition is NOT satisfied and cannot be satisfied, mark ${shouldIncludeSubGoals ? 'the goal' : 'it'} as failed (success=\"false\").\n- If the page is still loading (e.g., you see a loading spinner, skeleton screen, or progress bar), do NOT assert yet. Wait for the page to finish loading before evaluating the assertion.\n${\n  !shouldIncludeSubGoals\n    ? `\n**Page navigation restriction:**\n- Unless the user's instruction explicitly asks you to click a link, jump to another page, or navigate to a URL, you MUST complete the task on the current page only.\n- Do NOT navigate away from the current page on your own initiative (e.g., do not click links that lead to other pages, do not use browser back/forward, do not open new URLs).\n- If the task cannot be accomplished on the current page and the user has not instructed you to navigate, report it as a failure (success=\"false\") instead of attempting to navigate to other pages.\n`\n    : ''\n}\n### Output Rules\n\n- If the task is NOT complete, skip this section and continue to Step ${actionStepNumber}.\n- Use the <complete success=\"true|false\">message</complete> tag to output the result if the goal is accomplished or failed.\n  - the 'success' attribute is required. ${shouldIncludeSubGoals ? 'It means whether the expected goal is accomplished based on what you observe in the current screenshot. ' : ''}No matter what actions were executed or what errors occurred during execution, if the ${shouldIncludeSubGoals ? 'expected goal is accomplished' : 'instruction is fulfilled'}, set success=\"true\". If the ${shouldIncludeSubGoals ? 'expected goal is not accomplished and cannot be accomplished' : 'instruction is not fulfilled and cannot be fulfilled'}, set success=\"false\".\n  - the 'message' is the information that will be provided to the user. If the user asks for a specific format, strictly follow that.\n- If you output <complete>, do NOT output <action-type> or <action-param-json>. The task ends here.\n\n## Step ${actionStepNumber}: Determine Next Action (related tags: <log>, <action-type>, <action-param-json>, <error>)\n\nONLY if the task is not complete: Think what the next action is according to the current screenshot${shouldIncludeSubGoals ? ' and the plan' : ''}.\n\n- Don't give extra actions or plans beyond the instruction or the plan. For example, don't try to submit the form if the instruction is only to fill something.\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully. Otherwise, retry or do something else to recover.\n- Give just the next ONE action you should do (if any)\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 3 times, you should think this is an error and set the \"error\" field to the error message.\n\n### Supporting actions list\n\n${actionList}\n\n### Log to give user feedback (preamble message)\n\nThe <log> tag is a brief preamble message to the user explaining what you're about to do. It should follow these principles and examples:\n\n- **Use ${preferredLanguage}**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what's been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- <log>Click the login button</log>\n- <log>Scroll to find the 'Yes' button in popup</log>\n- <log>Previous actions failed to find the 'Yes' button, i will try again</log>\n- <log>Go back to find the login button</log>\n\n### If there is some action to do ...\n\n- Use the <action-type> and <action-param-json> tags to output the action to be executed.\n- The <action-type> MUST be one of the supporting actions. 'complete' is NOT a valid action-type.\n- Parameter names are strict. Use EXACTLY the field names listed for the selected action. Do NOT invent alias fields. If an action has a \"sample\" in its description, follow that structure.\nFor example:\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateExample1}\n}\n</action-param-json>\n\n### If you think there is an error ...\n\n- Use the <error> tag to output the error message.\n\nFor example:\n<error>Unable to find the required element on the page</error>\n\n### If there is no action to do ...\n\n- Don't output <action-type> or <action-param-json> if there is no action to do.\n\n## Return Format\n\nReturn in XML format following this decision flow:\n\n**Always include (REQUIRED):**\n<!-- Step 1: Observe${shouldIncludeSubGoals ? ' and Plan' : ''} -->\n<thought>Your thought process here. NEVER skip this tag.</thought>\n${\n  shouldIncludeSubGoals\n    ? `\n<!-- required when no update-plan-content is provided in the previous response -->\n<update-plan-content>...</update-plan-content>\n\n<!-- required when any sub-goal is completed -->\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n`\n    : ''\n}${\n  shouldIncludeSubGoals\n    ? `\n<!-- Step ${memoryStepNumber}: Memory data from current screenshot if needed -->\n<memory>...</memory>\n`\n    : ''\n}\n**Then choose ONE of the following paths:**\n\n**Path A: If the ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} or failed (Step ${checkGoalStepNumber})**\n<complete success=\"true|false\">...</complete>\n\n**Path B: If the ${shouldIncludeSubGoals ? 'goal is NOT complete' : 'instruction is NOT fulfilled'} yet (Step ${actionStepNumber})**\n<!-- Determine next action -->\n<log>...</log>\n<action-type>...</action-type>\n<action-param-json>...</action-param-json>\n\n<!-- OR if there's an error -->\n<error>...</error>\n${\n  shouldIncludeSubGoals\n    ? `\n## Multi-turn Conversation Example\n\nBelow is an example of a multi-turn conversation for \"fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address\":\n\n### Turn 1 - Initial instruction\n\n**User message:**\n<user_instruction>fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address</user_instruction>\n\n**Screenshot:** [Shows a registration form with empty Name and Email fields]\n\n**Your response:**\n<thought>The user wants me to fill out the registration form with specific values and return the email address. I can see the form has two fields: Name and Email. Both are currently empty. I'll break this down into sub-goals and start with the Name field. Note: The instruction is to fill the form only (not submit), and return the email at the end.</thought>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Fill in the Name field with 'John'</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Fill in the Email field with 'john@example.com'</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Return the filled email address</sub-goal>\n</update-plan-content>\n<log>Click on the Name field to start filling the form</log>\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateNameField}\n}\n</action-param-json>\n\n### Turn 2 - After clicking Name field\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\nSub-goals:\n1. Fill in the Name field with 'John' (running)\n2. Fill in the Email field with 'john@example.com' (pending)\n3. Return the filled email address (pending)\nCurrent sub-goal is: Fill in the Name field with 'John'\nActions performed for current sub-goal:\n- Click on the Name field to start filling the form\n\n**Screenshot:** [Shows the form with Name field now focused/active]\n\n**Your response:**\n<thought>The Name field is now focused. I need to type 'John' into this field. Current sub-goal is running, will be completed after input.</thought>\n<log>Typing 'John' into the Name field</log>\n<action-type>Input</action-type>\n<action-param-json>\n{\n  \"value\": \"John\"\n}\n</action-param-json>\n\n### Turn 3 - After entering name\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\nSub-goals:\n1. Fill in the Name field with 'John' (running)\n2. Fill in the Email field with 'john@example.com' (pending)\n3. Return the filled email address (pending)\nCurrent sub-goal is: Fill in the Name field with 'John'\nActions performed for current sub-goal:\n- Click on the Name field to start filling the form\n- Typing 'John' into the Name field\n\n**Screenshot:** [Shows the form with Name field containing 'John']\n\n**Your response:**\n<thought>The Name field now shows 'John'. Sub-goal 1 is complete. Next, I need to fill the Email field for sub-goal 2.</thought>\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n<memory>Name field has been filled with 'John'</memory>\n<log>Moving to the Email field</log>\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateEmailField}\n}\n</action-param-json>\n\n### Turn 4 - After clicking Email field\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\nSub-goals:\n1. Fill in the Name field with 'John' (finished)\n2. Fill in the Email field with 'john@example.com' (running)\n3. Return the filled email address (pending)\nCurrent sub-goal is: Fill in the Email field with 'john@example.com'\nActions performed for current sub-goal:\n- Moving to the Email field\n\n**Screenshot:** [Shows the form with Name='John' and Email field focused]\n\n**Your response:**\n<thought>The Email field is now focused. I'll enter 'john@example.com'. After this, sub-goal 2 will be complete and my task will be done.</thought>\n<log>Typing email address into the Email field</log>\n<action-type>Input</action-type>\n<action-param-json>\n{\n  \"value\": \"john@example.com\"\n}\n</action-param-json>\n\n### Turn 5 - After entering email (Goal accomplished)\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\nSub-goals:\n1. Fill in the Name field with 'John' (finished)\n2. Fill in the Email field with 'john@example.com' (running)\n3. Return the filled email address (pending)\nCurrent sub-goal is: Fill in the Email field with 'john@example.com'\nActions performed for current sub-goal:\n- Moving to the Email field\n- Typing email address into the Email field\n\n**Screenshot:** [Shows the form with Name='John' and Email='john@example.com']\n\n**Your response:**\n<thought>Both fields are now filled: Name shows 'John' and Email shows 'john@example.com'. Sub-goal 2 is complete. The user asked me to return the filled email address, so I need to include 'john@example.com' in my response. All sub-goals are now finished.</thought>\n<mark-sub-goal-done>\n  <sub-goal index=\"2\" status=\"finished\" />\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>\n<complete success=\"true\">john@example.com</complete>\n`\n    : `\n## Multi-turn Conversation Example\n\nBelow is an example of a multi-turn conversation for \"fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address\":\n\n### Turn 1 - Initial instruction\n\n**User message:**\n<user_instruction>fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address</user_instruction>\n\n**Screenshot:** [Shows a registration form with empty Name and Email fields]\n\n**Your response:**\n<thought>The user wants me to fill out the registration form with specific values and return the email address. I can see the form has two fields: Name and Email. Both are currently empty. I should start by clicking on the Name field. Note: The instruction is to fill the form only (not submit), and return the email at the end.</thought>\n<log>Click on the Name field to start filling the form</log>\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateNameField}\n}\n</action-param-json>\n\n### Turn 2 - After clicking Name field\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\n**Screenshot:** [Shows the form with Name field now focused/active]\n\n**Your response:**\n<thought>The Name field is now focused. I need to type 'John' into this field. After this, I'll need to fill the Email field as well.</thought>\n<log>Typing 'John' into the Name field</log>\n<action-type>Input</action-type>\n<action-param-json>\n{\n  \"value\": \"John\"\n}\n</action-param-json>\n\n### Turn 3 - After entering name\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\n**Screenshot:** [Shows the form with Name field containing 'John']\n\n**Your response:**\n<thought>The Name field now shows 'John'. Next, I need to fill the Email field with 'john@example.com'. I'll click on the Email field first.</thought>\n<log>Moving to the Email field</log>\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateEmailField}\n}\n</action-param-json>\n\n### Turn 4 - After clicking Email field\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\n**Screenshot:** [Shows the form with Name='John' and Email field focused]\n\n**Your response:**\n<thought>The Email field is now focused. I'll enter 'john@example.com'. After this, both required fields will be filled and my task will be complete.</thought>\n<log>Typing email address into the Email field</log>\n<action-type>Input</action-type>\n<action-param-json>\n{\n  \"value\": \"john@example.com\"\n}\n</action-param-json>\n\n### Turn 5 - After entering email (Instruction fulfilled)\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\n**Screenshot:** [Shows the form with Name='John' and Email='john@example.com']\n\n**Your response:**\n<thought>Both fields are now filled: Name shows 'John' and Email shows 'john@example.com'. The user asked me to return the filled email address, so I should include 'john@example.com' in my response. The instruction has been fulfilled.</thought>\n<complete success=\"true\">john@example.com</complete>\n`\n}`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","vlLocateParam","modelFamily","bboxDescription","findDefaultValue","field","current","visited","Set","currentWithDef","SAMPLE_BBOXES","injectBboxIntoSample","sample","locateFields","includeBbox","result","bboxIndex","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","isOptional","keyWithOptional","typeName","getZodTypeName","description","getZodDescription","defaultValue","hasDefault","undefined","paramLine","comments","defaultStr","JSON","line","paramDescription","findAllMidsceneLocatorField","sampleWithBbox","sampleStr","systemPromptToTaskPlanning","actionSpace","includeThought","includeSubGoals","preferredLanguage","getPreferredLanguage","Error","actionDescriptionList","actionList","shouldIncludeSubGoals","locateExample1","locateNameField","locateEmailField","step1Title","step1Description","explicitInstructionRule","thoughtTagDescription","subGoalTags","memoryStepNumber","checkGoalStepNumber","actionStepNumber"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACKA,MAAMI,gBAAgB,CAACC;IACrB,IAAIA,aACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EAAgBD,cAAc;IAEvG,OAAO;AACT;AAKA,MAAME,mBAAmB,CAACC;IACxB,IAAIC,UAAUD;IACd,MAAME,UAAU,IAAIC;IAEpB,MAAOF,WAAW,CAACC,QAAQ,GAAG,CAACD,SAAU;QACvCC,QAAQ,GAAG,CAACD;QACZ,MAAMG,iBAAiBH;QAQvB,IAAI,CAACG,eAAe,IAAI,EAAE,UAAU;QAEpC,IAAIA,AAAiC,iBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAC9B,OAAOA,eAAe,IAAI,CAAC,YAAY;QAIzC,IACEA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,IAC5BA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAE5BH,UAAUG,eAAe,IAAI,CAAC,SAAS;aAEvC;IAEJ;AAGF;AAOA,MAAMC,gBAAoD;IACxD;QAAC;QAAI;QAAK;QAAK;KAAI;IACnB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAI;QAAK;QAAK;KAAI;CACpB;AAED,MAAMC,uBAAuB,CAC3BC,QACAC,cACAC;IAEA,IAAI,CAACA,aAAa,OAAOF;IACzB,MAAMG,SAAS;QAAE,GAAGH,MAAM;IAAC;IAC3B,IAAII,YAAY;IAChB,KAAK,MAAMX,SAASQ,aAClB,IACEE,MAAM,CAACV,MAAM,IACb,AAAyB,YAAzB,OAAOU,MAAM,CAACV,MAAM,IACpBU,MAAM,CAACV,MAAM,CAAC,MAAM,EACpB;QACAU,MAAM,CAACV,MAAM,GAAG;YACd,GAAGU,MAAM,CAACV,MAAM;YAChB,MAAMK,aAAa,CAACM,YAAYN,cAAc,MAAM,CAAC;QACvD;QACAM;IACF;IAEF,OAAOD;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC,8BACAL,cAAc,KAAK;IAEnB,MAAMM,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QAIjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAE1B,KAAK,MAAM,CAAC3B,KAAKS,MAAM,IAAIR,OAAO,OAAO,CAAC4B,OACxC,IAAIpB,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMqB,aACJ,AACE,cADF,OAAQrB,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBACrD,MAAMsB,kBAAkBD,aAAa,GAAG9B,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMgC,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAexB,OAAOc;gBAGvC,MAAMW,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkB1B;gBAGtC,MAAM2B,eAAe5B,iBAAiBC;gBACtC,MAAM4B,aAAaD,AAAiBE,WAAjBF;gBAGnB,IAAIG,YAAY,GAAGR,gBAAgB,EAAE,EAAEC,UAAU;gBACjD,MAAMQ,WAAqB,EAAE;gBAC7B,IAAIN,aACFM,SAAS,IAAI,CAACN;gBAEhB,IAAIG,YAAY;oBACd,MAAMI,aACJ,AAAwB,YAAxB,OAAOL,eACH,CAAC,CAAC,EAAEA,aAAa,CAAC,CAAC,GACnBM,KAAK,SAAS,CAACN;oBACrBI,SAAS,IAAI,CAAC,CAAC,SAAS,EAAEC,YAAY;gBACxC;gBACA,IAAID,SAAS,MAAM,GAAG,GACpBD,aAAa,CAAC,IAAI,EAAEC,SAAS,IAAI,CAAC,OAAO;gBAG3Cd,WAAW,IAAI,CAACa;YAClB;YAIF,IAAIb,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACiB;oBAClBlB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEkB,MAAM;gBAC3B;YACF;QACF,OAAO;YAEL,MAAMX,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAeN;YAChC,MAAMO,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkBR;YAGtC,IAAIiB,mBAAmB,CAAC,SAAS,EAAEZ,UAAU;YAC7C,IAAIE,aACFU,oBAAoB,CAAC,IAAI,EAAEV,aAAa;YAE1CU,oBAAoB;YAEpBnB,OAAO,IAAI,CAACmB;QACd;IACF;IAGA,IAAItB,OAAO,MAAM,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EAAe;QACtD,MAAML,eAAe4B,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BvB,OAAO,WAAW;QACnE,MAAMwB,iBAAiB/B,qBACrBO,OAAO,MAAM,EACbL,cACAC;QAEF,MAAM6B,YAAY,CAAC,WAAW,EAAEvB,MAAMA,IAAI,aAAa,EAAEF,OAAO,IAAI,CAAC,gBAAgB,EAAEE,MAAMA,IAAI,qBAAqB,EAAEA,MAAMA,MAAMkB,KAAK,SAAS,CAACI,gBAAgB,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,EAAEtB,MAAMA,KAAK,EAAE,EAAE,EAAEA,MAAMA,IAAI,oBAAoB,CAAC;QAChPC,OAAO,IAAI,CAACsB;IACd;IAEA,OAAO,CAAC,EAAE,EAAEzB,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAewB,2BAA2B,EAC/CC,WAAW,EACX3C,WAAW,EACXY,WAAW,EACXgC,cAAc,EACdC,eAAe,EAOhB;IACC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAG1B,IAAInC,eAAe,CAACZ,aAClB,MAAM,IAAIgD,MACR;IAIJ,MAAMC,wBAAwBN,YAAY,GAAG,CAAC,CAAC3B,SACtCD,qBACLC,QACAjB,cAAca,cAAcZ,cAAcgC,SAC1CpB;IAGJ,MAAMsC,aAAaD,sBAAsB,IAAI,CAAC;IAG9C,MAAME,wBAAwBN,mBAAmB;IAGjD,MAAMO,iBAAiBxC,cACnB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAGF,MAAMyC,kBAAkBzC,cACpB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAM0C,mBAAmB1C,cACrB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAMF,MAAM2C,aAAaJ,wBACf,uGACA;IAEJ,MAAMK,mBAAmBL,wBACrB,wNACA;IAEJ,MAAMM,0BAA0B;IAEhC,MAAMC,wBAAwBP,wBAC1B,CAAC;;;;AAIP,EAAEM,yBAAyB,GACrB,CAAC;;;;AAIP,EAAEA,yBAAyB;IAEzB,MAAME,cAAcR,wBAChB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAuDc,CAAC,GAChB;IAIJ,MAAMS,mBAAmB;IACzB,MAAMC,sBAAsBV,wBAAwB,IAAI;IACxD,MAAMW,mBAAmBX,wBAAwB,IAAI;IAErD,OAAO,CAAC;;;AAGV,EAAEI,WAAW;;AAEb,EAAEC,iBAAiB;;;;AAInB,EAAEE,sBAAsB;AACxB,EAAEC,YAAY;AACd,EACER,wBACI,CAAC;QACC,EAAES,iBAAiB;;;;;AAK3B,CAAC,GACK,GACL;QACO,EAAEC,oBAAoB,EAAE,EAAEV,wBAAwB,kCAAkC,wCAAwC;;AAEpI,EAAEA,wBAAwB,+EAA+E,YAAY;;;;;;;;;;QAU7G,EAAEA,wBAAwB,sBAAsB,wBAAwB;MAC1E,EAAEA,wBAAwB,yBAAyB,2BAA2B;;;;wBAI5D,EAAEA,wBAAwB,sBAAsB,wBAAwB;6BACnE,EAAEA,wBAAwB,sBAAsB,wBAAwB;qCAChE,EAAEA,wBAAwB,sBAAsB,wBAAwB;4BACjF,EAAEA,wBAAwB,sBAAsB,wBAAwB;;;0NAGsH,EAAEA,wBAAwB,aAAa,KAAK;;AAEtQ,EACE,CAACA,wBACG,CAAC;;;;;AAKP,CAAC,GACK,GACL;;;sEAGqE,EAAEW,iBAAiB;;yCAEhD,EAAEX,wBAAwB,6GAA6G,GAAG,sFAAsF,EAAEA,wBAAwB,kCAAkC,2BAA2B,6BAA6B,EAAEA,wBAAwB,iEAAiE,uDAAuD;;;;QAIvgB,EAAEW,iBAAiB;;mGAEwE,EAAEX,wBAAwB,kBAAkB,GAAG;;;;;;;;;;AAUlJ,EAAED,WAAW;;;;;;QAML,EAAEJ,kBAAkB;;;;;;;;;;;;;;;;;;;;YAoBhB,EAAEM,eAAe;;;;;;;;;;;;;;;;;;;;oBAoBT,EAAED,wBAAwB,cAAc,GAAG;;AAE/D,EACEA,wBACI,CAAC;;;;;;;;AAQP,CAAC,GACK,KAEJA,wBACI,CAAC;UACG,EAAES,iBAAiB;;AAE7B,CAAC,GACK,GACL;;;iBAGgB,EAAET,wBAAwB,yBAAyB,2BAA2B,iBAAiB,EAAEU,oBAAoB;;;iBAGrH,EAAEV,wBAAwB,yBAAyB,+BAA+B,WAAW,EAAEW,iBAAiB;;;;;;;;AAQjI,EACEX,wBACI,CAAC;;;;;;;;;;;;;;;;;;;;;;;YAuBK,EAAEE,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAuDlB,EAAEC,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoD/B,CAAC,GACK,CAAC;;;;;;;;;;;;;;;;;;YAkBK,EAAED,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAkClB,EAAEC,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+B/B,CAAC,EACC;AACF"}
+{"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/types';\nimport type { TModelFamily } from '@aiscene/shared/env';\nimport { getPreferredLanguage } from '@aiscene/shared/env';\nimport {\n  getZodDescription,\n  getZodTypeName,\n} from '@aiscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { bboxDescription } from './common';\n\nconst vlLocateParam = (modelFamily: TModelFamily | undefined) => {\n  if (modelFamily) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(modelFamily)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\n/**\n * Find ZodDefault in the wrapper chain and return its default value\n */\nconst findDefaultValue = (field: unknown): any | undefined => {\n  let current = field;\n  const visited = new Set<unknown>();\n\n  while (current && !visited.has(current)) {\n    visited.add(current);\n    const currentWithDef = current as {\n      _def?: {\n        typeName?: string;\n        defaultValue?: () => any;\n        innerType?: unknown;\n      };\n    };\n\n    if (!currentWithDef._def?.typeName) break;\n\n    if (currentWithDef._def.typeName === 'ZodDefault') {\n      return currentWithDef._def.defaultValue?.();\n    }\n\n    // Continue unwrapping if it's a wrapper type\n    if (\n      currentWithDef._def.typeName === 'ZodOptional' ||\n      currentWithDef._def.typeName === 'ZodNullable'\n    ) {\n      current = currentWithDef._def.innerType;\n    } else {\n      break;\n    }\n  }\n\n  return undefined;\n};\n\n/**\n * Inject bbox into locate fields of a sample object.\n * Walks the sample and for any locate field (identified by paramSchema),\n * adds a fake bbox array when includeBbox is true.\n */\nconst SAMPLE_BBOXES: [number, number, number, number][] = [\n  [50, 100, 200, 200],\n  [300, 400, 500, 500],\n  [600, 100, 800, 250],\n  [50, 600, 250, 750],\n];\n\nconst injectBboxIntoSample = (\n  sample: Record<string, any>,\n  locateFields: string[],\n  includeBbox: boolean,\n): Record<string, any> => {\n  if (!includeBbox) return sample;\n  const result = { ...sample };\n  let bboxIndex = 0;\n  for (const field of locateFields) {\n    if (\n      result[field] &&\n      typeof result[field] === 'object' &&\n      result[field].prompt\n    ) {\n      result[field] = {\n        ...result[field],\n        bbox: SAMPLE_BBOXES[bboxIndex % SAMPLE_BBOXES.length],\n      };\n      bboxIndex++;\n    }\n  }\n  return result;\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n  includeBbox = false,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  // Add the action type field\n  fields.push(`- type: \"${action.name}\"`);\n\n  // Handle paramSchema if it exists\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    // Check if paramSchema is a ZodObject with shape\n    const schema = action.paramSchema as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      // Original logic for ZodObject schemas\n      const shape = schema.shape;\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          // Check if field is optional\n          const isOptional =\n            typeof (field as { isOptional?: () => boolean }).isOptional ===\n              'function' &&\n            (field as { isOptional: () => boolean }).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          // Get the type name using extracted helper\n          const typeName = getZodTypeName(field, locatorSchemaTypeDescription);\n\n          // Get description using extracted helper\n          const description = getZodDescription(field as z.ZodTypeAny);\n\n          // Check if field has a default value by searching the wrapper chain\n          const defaultValue = findDefaultValue(field);\n          const hasDefault = defaultValue !== undefined;\n\n          // Build param line for this field\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          const comments: string[] = [];\n          if (description) {\n            comments.push(description);\n          }\n          if (hasDefault) {\n            const defaultStr =\n              typeof defaultValue === 'string'\n                ? `\"${defaultValue}\"`\n                : JSON.stringify(defaultValue);\n            comments.push(`default: ${defaultStr}`);\n          }\n          if (comments.length > 0) {\n            paramLine += ` // ${comments.join(', ')}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      // Add the param section to fields if there are paramLines\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      // Handle non-object schemas (string, number, etc.)\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(schema as z.ZodTypeAny);\n\n      // For simple types, indicate that param should be the direct value, not an object\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  // Render sample if provided, using the same XML tag format as the real output\n  if (action.sample && typeof action.sample === 'object') {\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n    const sampleWithBbox = injectBboxIntoSample(\n      action.sample,\n      locateFields,\n      includeBbox,\n    );\n    const sampleStr = `- sample:\\n${tab}${tab}<action-type>${action.name}</action-type>\\n${tab}${tab}<action-param-json>\\n${tab}${tab}${JSON.stringify(sampleWithBbox, null, 2).replace(/\\n/g, `\\n${tab}${tab}`)}\\n${tab}${tab}</action-param-json>`;\n    fields.push(sampleStr);\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  modelFamily,\n  includeBbox,\n  includeThought,\n  includeSubGoals,\n}: {\n  actionSpace: DeviceAction<any>[];\n  modelFamily: TModelFamily | undefined;\n  includeBbox: boolean;\n  includeThought?: boolean;\n  includeSubGoals?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n\n  // Validate parameters: if includeBbox is true, modelFamily must be defined\n  if (includeBbox && !modelFamily) {\n    throw new Error(\n      'modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? modelFamily : undefined),\n      includeBbox,\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const shouldIncludeThought = includeThought ?? true;\n  const shouldIncludeSubGoals = includeSubGoals ?? false;\n\n  // Generate locate object examples based on includeBbox\n  const locateExample1 = includeBbox\n    ? `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\",\n    \"bbox\": [345, 442, 458, 483]\n  }`\n    : `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\"\n  }`;\n\n  // Locate examples for multi-turn conversation\n  const locateNameField = includeBbox\n    ? `{\n    \"prompt\": \"Name input field in the registration form\",\n    \"bbox\": [120, 180, 380, 210]\n  }`\n    : `{\n    \"prompt\": \"Name input field in the registration form\"\n  }`;\n\n  const locateEmailField = includeBbox\n    ? `{\n    \"prompt\": \"Email input field in the registration form\",\n    \"bbox\": [120, 240, 380, 270]\n  }`\n    : `{\n    \"prompt\": \"Email input field in the registration form\"\n  }`;\n\n  const thoughtTag = (content: string) =>\n    shouldIncludeThought ? `<thought>${content}</thought>\\n` : '';\n\n  // Sub-goals related content - only included when shouldIncludeSubGoals is true\n  const step1Title = shouldIncludeSubGoals\n    ? '## Step 1: Observe and Plan (related tags: <thought>, <update-plan-content>, <mark-sub-goal-done>)'\n    : '## Step 1: Observe (related tags: <thought>)';\n\n  const step1Description = shouldIncludeSubGoals\n    ? \"First, observe the current screenshot and previous logs, then break down the user's instruction into multiple high-level sub-goals. Update the status of sub-goals based on what you see in the current screenshot.\"\n    : 'First, observe the current screenshot and previous logs to understand the current state.';\n\n  const explicitInstructionRule = `CRITICAL - Following Explicit Instructions: When the user gives you specific operation steps (not high-level goals), you MUST execute ONLY those exact steps - nothing more, nothing less. Do NOT add extra actions even if they seem logical. For example: \"fill out the form\" means only fill fields, do NOT submit; \"click the button\" means only click, do NOT wait for page load or verify results; \"type 'hello'\" means only type, do NOT press Enter.`;\n\n  const thoughtTagDescription = shouldIncludeSubGoals\n    ? `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`\n    : `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the current state based on the screenshot? What should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`;\n\n  const subGoalTags = shouldIncludeSubGoals\n    ? `\n\n* <update-plan-content> tag\n\nUse this structure to give or update your plan:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished|pending\">sub goal description</sub-goal>\n  <sub-goal index=\"2\" status=\"finished|pending\">sub goal description</sub-goal>\n  ...\n</update-plan-content>\n\n* <mark-sub-goal-done> tag\n\nUse this structure to mark a sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nIMPORTANT: You MUST only mark a sub-goal as \"finished\" AFTER you have confirmed the task is actually completed by observing the result in the screenshot. Do NOT mark a sub-goal as done just because you expect the next action will complete it. Wait until you see visual confirmation in the screenshot that the sub-goal has been achieved.\n\n* Note\n\nDuring execution, you can call <update-plan-content> at any time to update the plan based on the latest screenshot and completed sub-goals.\n\n### Example\n\nIf the user wants to \"log in to a system using username and password, complete all to-do items, and submit a registration form\", you can break it down into the following sub-goals:\n\n<thought>...</thought>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Log in to the system</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Complete all to-do items</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Submit the registration form</sub-goal>\n</update-plan-content>\n\nAfter logging in and seeing the to-do items, you can mark the sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nAt this point, the status of all sub-goals is:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished\" />\n  <sub-goal index=\"2\" status=\"pending\" />\n  <sub-goal index=\"3\" status=\"pending\" />\n</update-plan-content>\n\nAfter some time, when the last sub-goal is also completed, you can mark it as done as well:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>`\n    : '';\n\n  // Step numbering adjusts based on whether sub-goals are included\n  // When includeSubGoals=false, memory step is skipped\n  const memoryStepNumber = 2; // Only used when shouldIncludeSubGoals is true\n  const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;\n  const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;\n\n  return `\nTarget: You are an expert to manipulate the UI to accomplish the user's instruction. User will give you an instruction, some screenshots, background knowledge and previous logs indicating what have been done. Your task is to accomplish the instruction by thinking through the path to complete the task and give the next action to execute.\n\n${step1Title}\n\n${step1Description}\n\n* <thought> tag (REQUIRED)\n\n${thoughtTagDescription}\n${subGoalTags}\n${\n  shouldIncludeSubGoals\n    ? `\n## Step ${memoryStepNumber}: Memory Data from Current Screenshot (related tags: <memory>)\n\nWhile observing the current screenshot, if you notice any information that might be needed in follow-up actions, record it here. The current screenshot will NOT be available in subsequent steps, so this memory is your only way to preserve essential information. Examples: extracted data, element states, content that needs to be referenced.\n\nDon't use this tag if no information needs to be preserved.\n`\n    : ''\n}\n## Step ${checkGoalStepNumber}: ${shouldIncludeSubGoals ? 'Check if Goal is Accomplished' : 'Check if the Instruction is Fulfilled'} (related tags: <complete>)\n\n${shouldIncludeSubGoals ? 'Based on the current screenshot and the status of all sub-goals, determine' : 'Determine'} if the entire task is completed.\n\n### CRITICAL: The User's Instruction is the Supreme Authority\n\nThe user's instruction defines the EXACT scope of what you must accomplish. You MUST follow it precisely - nothing more, nothing less. Violating this rule may cause severe consequences such as data loss, unintended operations, or system failures.\n\n**Explicit instructions vs. High-level goals:**\n- If the user gives you **explicit operation steps** (e.g., \"click X\", \"type Y\", \"fill out the form\"), treat them as exact commands. Execute ONLY those steps, nothing more.\n- If the user gives you a **high-level goal** (e.g., \"log in to the system\", \"complete the purchase\"), you may determine the necessary steps to achieve it.\n\n**What \"${shouldIncludeSubGoals ? 'goal accomplished' : 'instruction fulfilled'}\" means:**\n- The ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} when you have done EXACTLY what the user asked - no extra steps, no assumptions.\n- Do NOT perform any action beyond the explicit instruction, even if it seems logical or helpful.\n\n**Examples - Explicit instructions (execute exactly, no extra steps):**\n- \"fill out the form\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when all fields are filled. Do NOT submit the form.\n- \"click the login button\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} once clicked. Do NOT wait for page load or verify login success.\n- \"type 'hello' in the search box\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when 'hello' is typed. Do NOT press Enter or trigger search.\n- \"select the first item\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when selected. Do NOT proceed to checkout.\n\n**Special case - Assertion instructions:**\n- If the user's instruction includes an assertion (e.g., \"verify that...\", \"check that...\", \"assert...\"), and you observe from the screenshot that the assertion condition is NOT satisfied and cannot be satisfied, mark ${shouldIncludeSubGoals ? 'the goal' : 'it'} as failed (success=\"false\").\n- If the page is still loading (e.g., you see a loading spinner, skeleton screen, or progress bar), do NOT assert yet. Wait for the page to finish loading before evaluating the assertion.\n${\n  !shouldIncludeSubGoals\n    ? `\n**Page navigation restriction:**\n- Unless the user's instruction explicitly asks you to click a link, jump to another page, or navigate to a URL, you MUST complete the task on the current page only.\n- Do NOT navigate away from the current page on your own initiative (e.g., do not click links that lead to other pages, do not use browser back/forward, do not open new URLs).\n- If the task cannot be accomplished on the current page and the user has not instructed you to navigate, report it as a failure (success=\"false\") instead of attempting to navigate to other pages.\n`\n    : ''\n}\n### Output Rules\n\n- If the task is NOT complete, skip this section and continue to Step ${actionStepNumber}.\n- Use the <complete success=\"true|false\">message</complete> tag to output the result if the goal is accomplished or failed.\n  - the 'success' attribute is required. ${shouldIncludeSubGoals ? 'It means whether the expected goal is accomplished based on what you observe in the current screenshot. ' : ''}No matter what actions were executed or what errors occurred during execution, if the ${shouldIncludeSubGoals ? 'expected goal is accomplished' : 'instruction is fulfilled'}, set success=\"true\". If the ${shouldIncludeSubGoals ? 'expected goal is not accomplished and cannot be accomplished' : 'instruction is not fulfilled and cannot be fulfilled'}, set success=\"false\".\n  - the 'message' is the information that will be provided to the user. If the user asks for a specific format, strictly follow that.\n- If you output <complete>, do NOT output <action-type> or <action-param-json>. The task ends here.\n\n## Step ${actionStepNumber}: Determine Next Action (related tags: <log>, <action-type>, <action-param-json>, <error>)\n\nONLY if the task is not complete: Think what the next action is according to the current screenshot${shouldIncludeSubGoals ? ' and the plan' : ''}.\n\n- Don't give extra actions or plans beyond the instruction or the plan. For example, don't try to submit the form if the instruction is only to fill something.\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully. Otherwise, retry or do something else to recover.\n- Give just the next ONE action you should do (if any)\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 3 times, you should think this is an error and set the \"error\" field to the error message.\n\n### Supporting actions list\n\n${actionList}\n\n### Log to give user feedback (preamble message)\n\nThe <log> tag is a brief preamble message to the user explaining what you're about to do. It should follow these principles and examples:\n\n- **Use ${preferredLanguage}**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what's been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- <log>Click the login button</log>\n- <log>Scroll to find the 'Yes' button in popup</log>\n- <log>Previous actions failed to find the 'Yes' button, i will try again</log>\n- <log>Go back to find the login button</log>\n\n### If there is some action to do ...\n\n- Use the <action-type> and <action-param-json> tags to output the action to be executed.\n- The <action-type> MUST be one of the supporting actions. 'complete' is NOT a valid action-type.\n- Parameter names are strict. Use EXACTLY the field names listed for the selected action. Do NOT invent alias fields. If an action has a \"sample\" in its description, follow that structure.\nFor example:\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateExample1}\n}\n</action-param-json>\n\n### If you think there is an error ...\n\n- Use the <error> tag to output the error message.\n\nFor example:\n<error>Unable to find the required element on the page</error>\n\n### If there is no action to do ...\n\n- Don't output <action-type> or <action-param-json> if there is no action to do.\n\n## Return Format\n\nReturn in XML format following this decision flow:\n\n**Always include (REQUIRED):**\n<!-- Step 1: Observe${shouldIncludeSubGoals ? ' and Plan' : ''} -->\n<thought>Your thought process here. NEVER skip this tag.</thought>\n${\n  shouldIncludeSubGoals\n    ? `\n<!-- required when no update-plan-content is provided in the previous response -->\n<update-plan-content>...</update-plan-content>\n\n<!-- required when any sub-goal is completed -->\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n`\n    : ''\n}${\n  shouldIncludeSubGoals\n    ? `\n<!-- Step ${memoryStepNumber}: Memory data from current screenshot if needed -->\n<memory>...</memory>\n`\n    : ''\n}\n**Then choose ONE of the following paths:**\n\n**Path A: If the ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} or failed (Step ${checkGoalStepNumber})**\n<complete success=\"true|false\">...</complete>\n\n**Path B: If the ${shouldIncludeSubGoals ? 'goal is NOT complete' : 'instruction is NOT fulfilled'} yet (Step ${actionStepNumber})**\n<!-- Determine next action -->\n<log>...</log>\n<action-type>...</action-type>\n<action-param-json>...</action-param-json>\n\n<!-- OR if there's an error -->\n<error>...</error>\n${\n  shouldIncludeSubGoals\n    ? `\n## Multi-turn Conversation Example\n\nBelow is an example of a multi-turn conversation for \"fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address\":\n\n### Turn 1 - Initial instruction\n\n**User message:**\n<user_instruction>fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address</user_instruction>\n\n**Screenshot:** [Shows a registration form with empty Name and Email fields]\n\n**Your response:**\n<thought>The user wants me to fill out the registration form with specific values and return the email address. I can see the form has two fields: Name and Email. Both are currently empty. I'll break this down into sub-goals and start with the Name field. Note: The instruction is to fill the form only (not submit), and return the email at the end.</thought>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Fill in the Name field with 'John'</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Fill in the Email field with 'john@example.com'</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Return the filled email address</sub-goal>\n</update-plan-content>\n<log>Click on the Name field to start filling the form</log>\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateNameField}\n}\n</action-param-json>\n\n### Turn 2 - After clicking Name field\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\nSub-goals:\n1. Fill in the Name field with 'John' (running)\n2. Fill in the Email field with 'john@example.com' (pending)\n3. Return the filled email address (pending)\nCurrent sub-goal is: Fill in the Name field with 'John'\nActions performed for current sub-goal:\n- Click on the Name field to start filling the form\n\n**Screenshot:** [Shows the form with Name field now focused/active]\n\n**Your response:**\n<thought>The Name field is now focused. I need to type 'John' into this field. Current sub-goal is running, will be completed after input.</thought>\n<log>Typing 'John' into the Name field</log>\n<action-type>Input</action-type>\n<action-param-json>\n{\n  \"value\": \"John\"\n}\n</action-param-json>\n\n### Turn 3 - After entering name\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\nSub-goals:\n1. Fill in the Name field with 'John' (running)\n2. Fill in the Email field with 'john@example.com' (pending)\n3. Return the filled email address (pending)\nCurrent sub-goal is: Fill in the Name field with 'John'\nActions performed for current sub-goal:\n- Click on the Name field to start filling the form\n- Typing 'John' into the Name field\n\n**Screenshot:** [Shows the form with Name field containing 'John']\n\n**Your response:**\n<thought>The Name field now shows 'John'. Sub-goal 1 is complete. Next, I need to fill the Email field for sub-goal 2.</thought>\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n<memory>Name field has been filled with 'John'</memory>\n<log>Moving to the Email field</log>\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateEmailField}\n}\n</action-param-json>\n\n### Turn 4 - After clicking Email field\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\nSub-goals:\n1. Fill in the Name field with 'John' (finished)\n2. Fill in the Email field with 'john@example.com' (running)\n3. Return the filled email address (pending)\nCurrent sub-goal is: Fill in the Email field with 'john@example.com'\nActions performed for current sub-goal:\n- Moving to the Email field\n\n**Screenshot:** [Shows the form with Name='John' and Email field focused]\n\n**Your response:**\n<thought>The Email field is now focused. I'll enter 'john@example.com'. After this, sub-goal 2 will be complete and my task will be done.</thought>\n<log>Typing email address into the Email field</log>\n<action-type>Input</action-type>\n<action-param-json>\n{\n  \"value\": \"john@example.com\"\n}\n</action-param-json>\n\n### Turn 5 - After entering email (Goal accomplished)\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\nSub-goals:\n1. Fill in the Name field with 'John' (finished)\n2. Fill in the Email field with 'john@example.com' (running)\n3. Return the filled email address (pending)\nCurrent sub-goal is: Fill in the Email field with 'john@example.com'\nActions performed for current sub-goal:\n- Moving to the Email field\n- Typing email address into the Email field\n\n**Screenshot:** [Shows the form with Name='John' and Email='john@example.com']\n\n**Your response:**\n<thought>Both fields are now filled: Name shows 'John' and Email shows 'john@example.com'. Sub-goal 2 is complete. The user asked me to return the filled email address, so I need to include 'john@example.com' in my response. All sub-goals are now finished.</thought>\n<mark-sub-goal-done>\n  <sub-goal index=\"2\" status=\"finished\" />\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>\n<complete success=\"true\">john@example.com</complete>\n`\n    : `\n## Multi-turn Conversation Example\n\nBelow is an example of a multi-turn conversation for \"fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address\":\n\n### Turn 1 - Initial instruction\n\n**User message:**\n<user_instruction>fill out the registration form with name 'John' and email 'john@example.com', then return the filled email address</user_instruction>\n\n**Screenshot:** [Shows a registration form with empty Name and Email fields]\n\n**Your response:**\n<thought>The user wants me to fill out the registration form with specific values and return the email address. I can see the form has two fields: Name and Email. Both are currently empty. I should start by clicking on the Name field. Note: The instruction is to fill the form only (not submit), and return the email at the end.</thought>\n<log>Click on the Name field to start filling the form</log>\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateNameField}\n}\n</action-param-json>\n\n### Turn 2 - After clicking Name field\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\n**Screenshot:** [Shows the form with Name field now focused/active]\n\n**Your response:**\n<thought>The Name field is now focused. I need to type 'John' into this field. After this, I'll need to fill the Email field as well.</thought>\n<log>Typing 'John' into the Name field</log>\n<action-type>Input</action-type>\n<action-param-json>\n{\n  \"value\": \"John\"\n}\n</action-param-json>\n\n### Turn 3 - After entering name\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\n**Screenshot:** [Shows the form with Name field containing 'John']\n\n**Your response:**\n<thought>The Name field now shows 'John'. Next, I need to fill the Email field with 'john@example.com'. I'll click on the Email field first.</thought>\n<log>Moving to the Email field</log>\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateEmailField}\n}\n</action-param-json>\n\n### Turn 4 - After clicking Email field\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\n**Screenshot:** [Shows the form with Name='John' and Email field focused]\n\n**Your response:**\n<thought>The Email field is now focused. I'll enter 'john@example.com'. After this, both required fields will be filled and my task will be complete.</thought>\n<log>Typing email address into the Email field</log>\n<action-type>Input</action-type>\n<action-param-json>\n{\n  \"value\": \"john@example.com\"\n}\n</action-param-json>\n\n### Turn 5 - After entering email (Instruction fulfilled)\n\n**User message:**\nThe previous action has been executed, here is the latest screenshot. Please continue according to the instruction.\n\n**Screenshot:** [Shows the form with Name='John' and Email='john@example.com']\n\n**Your response:**\n<thought>Both fields are now filled: Name shows 'John' and Email shows 'john@example.com'. The user asked me to return the filled email address, so I should include 'john@example.com' in my response. The instruction has been fulfilled.</thought>\n<complete success=\"true\">john@example.com</complete>\n`\n}`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","vlLocateParam","modelFamily","bboxDescription","findDefaultValue","field","current","visited","Set","currentWithDef","SAMPLE_BBOXES","injectBboxIntoSample","sample","locateFields","includeBbox","result","bboxIndex","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","isOptional","keyWithOptional","typeName","getZodTypeName","description","getZodDescription","defaultValue","hasDefault","undefined","paramLine","comments","defaultStr","JSON","line","paramDescription","findAllMidsceneLocatorField","sampleWithBbox","sampleStr","systemPromptToTaskPlanning","actionSpace","includeThought","includeSubGoals","preferredLanguage","getPreferredLanguage","Error","actionDescriptionList","actionList","shouldIncludeSubGoals","locateExample1","locateNameField","locateEmailField","step1Title","step1Description","explicitInstructionRule","thoughtTagDescription","subGoalTags","memoryStepNumber","checkGoalStepNumber","actionStepNumber"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACKA,MAAMI,gBAAgB,CAACC;IACrB,IAAIA,aACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EAAgBD,cAAc;IAEvG,OAAO;AACT;AAKA,MAAME,mBAAmB,CAACC;IACxB,IAAIC,UAAUD;IACd,MAAME,UAAU,IAAIC;IAEpB,MAAOF,WAAW,CAACC,QAAQ,GAAG,CAACD,SAAU;QACvCC,QAAQ,GAAG,CAACD;QACZ,MAAMG,iBAAiBH;QAQvB,IAAI,CAACG,eAAe,IAAI,EAAE,UAAU;QAEpC,IAAIA,AAAiC,iBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAC9B,OAAOA,eAAe,IAAI,CAAC,YAAY;QAIzC,IACEA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,IAC5BA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAE5BH,UAAUG,eAAe,IAAI,CAAC,SAAS;aAEvC;IAEJ;AAGF;AAOA,MAAMC,gBAAoD;IACxD;QAAC;QAAI;QAAK;QAAK;KAAI;IACnB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAI;QAAK;QAAK;KAAI;CACpB;AAED,MAAMC,uBAAuB,CAC3BC,QACAC,cACAC;IAEA,IAAI,CAACA,aAAa,OAAOF;IACzB,MAAMG,SAAS;QAAE,GAAGH,MAAM;IAAC;IAC3B,IAAII,YAAY;IAChB,KAAK,MAAMX,SAASQ,aAClB,IACEE,MAAM,CAACV,MAAM,IACb,AAAyB,YAAzB,OAAOU,MAAM,CAACV,MAAM,IACpBU,MAAM,CAACV,MAAM,CAAC,MAAM,EACpB;QACAU,MAAM,CAACV,MAAM,GAAG;YACd,GAAGU,MAAM,CAACV,MAAM;YAChB,MAAMK,aAAa,CAACM,YAAYN,cAAc,MAAM,CAAC;QACvD;QACAM;IACF;IAEF,OAAOD;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC,8BACAL,cAAc,KAAK;IAEnB,MAAMM,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QAIjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAE1B,KAAK,MAAM,CAAC3B,KAAKS,MAAM,IAAIR,OAAO,OAAO,CAAC4B,OACxC,IAAIpB,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMqB,aACJ,AACE,cADF,OAAQrB,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBACrD,MAAMsB,kBAAkBD,aAAa,GAAG9B,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMgC,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAexB,OAAOc;gBAGvC,MAAMW,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkB1B;gBAGtC,MAAM2B,eAAe5B,iBAAiBC;gBACtC,MAAM4B,aAAaD,AAAiBE,WAAjBF;gBAGnB,IAAIG,YAAY,GAAGR,gBAAgB,EAAE,EAAEC,UAAU;gBACjD,MAAMQ,WAAqB,EAAE;gBAC7B,IAAIN,aACFM,SAAS,IAAI,CAACN;gBAEhB,IAAIG,YAAY;oBACd,MAAMI,aACJ,AAAwB,YAAxB,OAAOL,eACH,CAAC,CAAC,EAAEA,aAAa,CAAC,CAAC,GACnBM,KAAK,SAAS,CAACN;oBACrBI,SAAS,IAAI,CAAC,CAAC,SAAS,EAAEC,YAAY;gBACxC;gBACA,IAAID,SAAS,MAAM,GAAG,GACpBD,aAAa,CAAC,IAAI,EAAEC,SAAS,IAAI,CAAC,OAAO;gBAG3Cd,WAAW,IAAI,CAACa;YAClB;YAIF,IAAIb,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACiB;oBAClBlB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEkB,MAAM;gBAC3B;YACF;QACF,OAAO;YAEL,MAAMX,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAeN;YAChC,MAAMO,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkBR;YAGtC,IAAIiB,mBAAmB,CAAC,SAAS,EAAEZ,UAAU;YAC7C,IAAIE,aACFU,oBAAoB,CAAC,IAAI,EAAEV,aAAa;YAE1CU,oBAAoB;YAEpBnB,OAAO,IAAI,CAACmB;QACd;IACF;IAGA,IAAItB,OAAO,MAAM,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EAAe;QACtD,MAAML,eAAe4B,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BvB,OAAO,WAAW;QACnE,MAAMwB,iBAAiB/B,qBACrBO,OAAO,MAAM,EACbL,cACAC;QAEF,MAAM6B,YAAY,CAAC,WAAW,EAAEvB,MAAMA,IAAI,aAAa,EAAEF,OAAO,IAAI,CAAC,gBAAgB,EAAEE,MAAMA,IAAI,qBAAqB,EAAEA,MAAMA,MAAMkB,KAAK,SAAS,CAACI,gBAAgB,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,EAAEtB,MAAMA,KAAK,EAAE,EAAE,EAAEA,MAAMA,IAAI,oBAAoB,CAAC;QAChPC,OAAO,IAAI,CAACsB;IACd;IAEA,OAAO,CAAC,EAAE,EAAEzB,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAewB,2BAA2B,EAC/CC,WAAW,EACX3C,WAAW,EACXY,WAAW,EACXgC,cAAc,EACdC,eAAe,EAOhB;IACC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAG1B,IAAInC,eAAe,CAACZ,aAClB,MAAM,IAAIgD,MACR;IAIJ,MAAMC,wBAAwBN,YAAY,GAAG,CAAC,CAAC3B,SACtCD,qBACLC,QACAjB,cAAca,cAAcZ,cAAcgC,SAC1CpB;IAGJ,MAAMsC,aAAaD,sBAAsB,IAAI,CAAC;IAG9C,MAAME,wBAAwBN,mBAAmB;IAGjD,MAAMO,iBAAiBxC,cACnB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAGF,MAAMyC,kBAAkBzC,cACpB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAM0C,mBAAmB1C,cACrB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAMF,MAAM2C,aAAaJ,wBACf,uGACA;IAEJ,MAAMK,mBAAmBL,wBACrB,wNACA;IAEJ,MAAMM,0BAA0B;IAEhC,MAAMC,wBAAwBP,wBAC1B,CAAC;;;;AAIP,EAAEM,yBAAyB,GACrB,CAAC;;;;AAIP,EAAEA,yBAAyB;IAEzB,MAAME,cAAcR,wBAChB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAuDc,CAAC,GAChB;IAIJ,MAAMS,mBAAmB;IACzB,MAAMC,sBAAsBV,wBAAwB,IAAI;IACxD,MAAMW,mBAAmBX,wBAAwB,IAAI;IAErD,OAAO,CAAC;;;AAGV,EAAEI,WAAW;;AAEb,EAAEC,iBAAiB;;;;AAInB,EAAEE,sBAAsB;AACxB,EAAEC,YAAY;AACd,EACER,wBACI,CAAC;QACC,EAAES,iBAAiB;;;;;AAK3B,CAAC,GACK,GACL;QACO,EAAEC,oBAAoB,EAAE,EAAEV,wBAAwB,kCAAkC,wCAAwC;;AAEpI,EAAEA,wBAAwB,+EAA+E,YAAY;;;;;;;;;;QAU7G,EAAEA,wBAAwB,sBAAsB,wBAAwB;MAC1E,EAAEA,wBAAwB,yBAAyB,2BAA2B;;;;wBAI5D,EAAEA,wBAAwB,sBAAsB,wBAAwB;6BACnE,EAAEA,wBAAwB,sBAAsB,wBAAwB;qCAChE,EAAEA,wBAAwB,sBAAsB,wBAAwB;4BACjF,EAAEA,wBAAwB,sBAAsB,wBAAwB;;;0NAGsH,EAAEA,wBAAwB,aAAa,KAAK;;AAEtQ,EACE,CAACA,wBACG,CAAC;;;;;AAKP,CAAC,GACK,GACL;;;sEAGqE,EAAEW,iBAAiB;;yCAEhD,EAAEX,wBAAwB,6GAA6G,GAAG,sFAAsF,EAAEA,wBAAwB,kCAAkC,2BAA2B,6BAA6B,EAAEA,wBAAwB,iEAAiE,uDAAuD;;;;QAIvgB,EAAEW,iBAAiB;;mGAEwE,EAAEX,wBAAwB,kBAAkB,GAAG;;;;;;;;;;AAUlJ,EAAED,WAAW;;;;;;QAML,EAAEJ,kBAAkB;;;;;;;;;;;;;;;;;;;;YAoBhB,EAAEM,eAAe;;;;;;;;;;;;;;;;;;;;oBAoBT,EAAED,wBAAwB,cAAc,GAAG;;AAE/D,EACEA,wBACI,CAAC;;;;;;;;AAQP,CAAC,GACK,KAEJA,wBACI,CAAC;UACG,EAAES,iBAAiB;;AAE7B,CAAC,GACK,GACL;;;iBAGgB,EAAET,wBAAwB,yBAAyB,2BAA2B,iBAAiB,EAAEU,oBAAoB;;;iBAGrH,EAAEV,wBAAwB,yBAAyB,+BAA+B,WAAW,EAAEW,iBAAiB;;;;;;;;AAQjI,EACEX,wBACI,CAAC;;;;;;;;;;;;;;;;;;;;;;;YAuBK,EAAEE,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAuDlB,EAAEC,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoD/B,CAAC,GACK,CAAC;;;;;;;;;;;;;;;;;;YAkBK,EAAED,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAkClB,EAAEC,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+B/B,CAAC,EACC;AACF"}

package/dist/lib/ai-model/prompt/llm-section-locator.js

@@ -27,7 +27,7 @@ __webpack_require__.d(__webpack_exports__, {
     sectionLocatorInstruction: ()=>sectionLocatorInstruction,
     systemPromptToLocateSection: ()=>systemPromptToLocateSection
 });
-const env_namespaceObject = require("@midscene/shared/env");
+const env_namespaceObject = require("@aiscene/shared/env");
 const external_common_js_namespaceObject = require("./common.js");
 function systemPromptToLocateSection(modelFamily) {
     const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();

package/dist/lib/ai-model/prompt/llm-section-locator.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/llm-section-locator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-section-locator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { bboxDescription } from './common';\n\nexport function systemPromptToLocateSection(\n  modelFamily: TModelFamily | undefined,\n) {\n  const preferredLanguage = getPreferredLanguage();\n  const bboxFormat = bboxDescription(modelFamily);\n  return `\n## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Find a section containing the target element\n- If the description mentions reference elements, also locate sections containing those references\n\n## Output Format:\n\\`\\`\\`json\n{\n  \"bbox\": [number, number, number, number],  // ${bboxFormat}\n  \"references_bbox\"?: [\n    [number, number, number, number],\n    ...\n  ],\n  \"error\"?: string\n}\n\\`\\`\\`\n\nFields:\n* \\`bbox\\` - Bounding box of the section containing the target element\n* \\`references_bbox\\` - Optional array of bounding boxes for reference elements\n* \\`error\\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.\n\nExample:\nIf the description is \"delete button on the second row with title 'Peter'\", return:\n\\`\\`\\`json\n{\n  \"bbox\": [100, 100, 200, 200],\n  \"references_bbox\": [[100, 100, 200, 200]]\n}\n\\`\\`\\`\n`;\n}\n\nexport const sectionLocatorInstruction = (sectionDescription: string) =>\n  `Find section containing: ${sectionDescription}`;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","systemPromptToLocateSection","modelFamily","preferredLanguage","getPreferredLanguage","bboxFormat","bboxDescription","sectionLocatorInstruction","sectionDescription"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACFO,SAASI,4BACdC,WAAqC;IAErC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAC1B,MAAMC,aAAaC,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EAAgBJ;IACnC,OAAO,CAAC;;;;;;;;;;;gDAWsC,EAAEG,WAAW;;;;;;;;;;;;yEAYY,EAAEF,kBAAkB;;;;;;;;;;AAU7F,CAAC;AACD;AAEO,MAAMI,4BAA4B,CAACC,qBACxC,CAAC,yBAAyB,EAAEA,oBAAoB"}
+{"version":3,"file":"ai-model/prompt/llm-section-locator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-section-locator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@aiscene/shared/env';\nimport { getPreferredLanguage } from '@aiscene/shared/env';\nimport { bboxDescription } from './common';\n\nexport function systemPromptToLocateSection(\n  modelFamily: TModelFamily | undefined,\n) {\n  const preferredLanguage = getPreferredLanguage();\n  const bboxFormat = bboxDescription(modelFamily);\n  return `\n## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Find a section containing the target element\n- If the description mentions reference elements, also locate sections containing those references\n\n## Output Format:\n\\`\\`\\`json\n{\n  \"bbox\": [number, number, number, number],  // ${bboxFormat}\n  \"references_bbox\"?: [\n    [number, number, number, number],\n    ...\n  ],\n  \"error\"?: string\n}\n\\`\\`\\`\n\nFields:\n* \\`bbox\\` - Bounding box of the section containing the target element\n* \\`references_bbox\\` - Optional array of bounding boxes for reference elements\n* \\`error\\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.\n\nExample:\nIf the description is \"delete button on the second row with title 'Peter'\", return:\n\\`\\`\\`json\n{\n  \"bbox\": [100, 100, 200, 200],\n  \"references_bbox\": [[100, 100, 200, 200]]\n}\n\\`\\`\\`\n`;\n}\n\nexport const sectionLocatorInstruction = (sectionDescription: string) =>\n  `Find section containing: ${sectionDescription}`;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","systemPromptToLocateSection","modelFamily","preferredLanguage","getPreferredLanguage","bboxFormat","bboxDescription","sectionLocatorInstruction","sectionDescription"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACFO,SAASI,4BACdC,WAAqC;IAErC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAC1B,MAAMC,aAAaC,AAAAA,IAAAA,mCAAAA,eAAAA,AAAAA,EAAgBJ;IACnC,OAAO,CAAC;;;;;;;;;;;gDAWsC,EAAEG,WAAW;;;;;;;;;;;;yEAYY,EAAEF,kBAAkB;;;;;;;;;;AAU7F,CAAC;AACD;AAEO,MAAMI,4BAA4B,CAACC,qBACxC,CAAC,yBAAyB,EAAEA,oBAAoB"}

package/dist/lib/ai-model/prompt/playwright-generator.js

@@ -35,7 +35,7 @@ __webpack_require__.d(__webpack_exports__, {
     prepareEventSummary: ()=>external_yaml_generator_js_namespaceObject.prepareEventSummary,
     processEventsForLLM: ()=>external_yaml_generator_js_namespaceObject.processEventsForLLM
 });
-const constants_namespaceObject = require("@midscene/shared/constants");
+const constants_namespaceObject = require("@aiscene/shared/constants");
 const external_index_js_namespaceObject = require("../index.js");
 const external_yaml_generator_js_namespaceObject = require("./yaml-generator.js");
 const generatePlaywrightTest = async (events, options, modelConfig)=>{

package/dist/lib/ai-model/prompt/playwright-generator.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/playwright-generator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/playwright-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { callAI, callAIWithStringResponse } from '../index';\n// Import shared utilities and types from yaml-generator\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './yaml-generator';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(prompt, modelConfig);\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAI(prompt, modelConfig, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(prompt, modelConfig);\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","generatePlaywrightTest","events","options","modelConfig","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","Error","generatePlaywrightTestStream","callAI"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;ACuDO,MAAMI,yBAAyB,OACpCC,QACAC,SACAC;IAGAC,IAAAA,2CAAAA,cAAAA,AAAAA,EAAeH;IAGf,MAAMI,UAAUC,AAAAA,IAAAA,2CAAAA,mBAAAA,AAAAA,EAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;8LAWiJ,CAAC;IAG7L,MAAMK,iBAAiBC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,QAAQb;IAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIE,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CnB,QACAC,SACAC;IAGAC,IAAAA,2CAAAA,cAAAA,AAAAA,EAAeH;IAGf,MAAMI,UAAUC,AAAAA,IAAAA,2CAAAA,mBAAAA,AAAAA,EAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;8LAYiJ,CAAC;IAG7L,MAAMK,iBAAiBC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMmB,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOL,QAAQb,aAAa;QACvC,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,QAAQb;QAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB;AACF"}
+{"version":3,"file":"ai-model/prompt/playwright-generator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/playwright-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@aiscene/shared/constants';\nimport type { IModelConfig } from '@aiscene/shared/env';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { callAI, callAIWithStringResponse } from '../index';\n// Import shared utilities and types from yaml-generator\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './yaml-generator';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(prompt, modelConfig);\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAI(prompt, modelConfig, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(prompt, modelConfig);\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","generatePlaywrightTest","events","options","modelConfig","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","Error","generatePlaywrightTestStream","callAI"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;ACuDO,MAAMI,yBAAyB,OACpCC,QACAC,SACAC;IAGAC,IAAAA,2CAAAA,cAAAA,AAAAA,EAAeH;IAGf,MAAMI,UAAUC,AAAAA,IAAAA,2CAAAA,mBAAAA,AAAAA,EAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;8LAWiJ,CAAC;IAG7L,MAAMK,iBAAiBC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,QAAQb;IAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIE,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CnB,QACAC,SACAC;IAGAC,IAAAA,2CAAAA,cAAAA,AAAAA,EAAeH;IAGf,MAAMI,UAAUC,AAAAA,IAAAA,2CAAAA,mBAAAA,AAAAA,EAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;8LAYiJ,CAAC;IAG7L,MAAMK,iBAAiBC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMmB,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOL,QAAQb,aAAa;QACvC,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,QAAQb;QAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB;AACF"}

package/dist/lib/ai-model/prompt/ui-tars-planning.js

@@ -27,7 +27,7 @@ __webpack_require__.d(__webpack_exports__, {
     getUiTarsPlanningPrompt: ()=>getUiTarsPlanningPrompt,
     getSummary: ()=>getSummary
 });
-const env_namespaceObject = require("@midscene/shared/env");
+const env_namespaceObject = require("@aiscene/shared/env");
 function getUiTarsPlanningPrompt() {
     const preferredLanguage = (0, env_namespaceObject.getPreferredLanguage)();
     return `

package/dist/lib/ai-model/prompt/ui-tars-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/ui-tars-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/ui-tars-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport function getUiTarsPlanningPrompt(): string {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. \n\n## Output Format\n\\`\\`\\`\nThought: ...\nAction: ...\n\\`\\`\\`\n\n## Action Space\n\nclick(start_box='[x1, y1, x2, y2]')\nleft_double(start_box='[x1, y1, x2, y2]')\nright_single(start_box='[x1, y1, x2, y2]')\ndrag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')\nhotkey(key='')\ntype(content='xxx') # Use escape characters \\\\', \\\\\\\", and \\\\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\\\n at the end of content. \nscroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')\nwait() #Sleep for 5s and take a screenshot to check for any changes.\nfinished(content='xxx') # Use escape characters \\\\', \\\\\", and \\\\n in content part to ensure we can parse the content in normal python string format.\n\n\n## Note\n- Use ${preferredLanguage} in \\`Thought\\` part.\n- Write a small plan and finally summarize your next action (with its target element) in one sentence in \\`Thought\\` part.\n\n## User Instruction\n`;\n}\n\nexport const getSummary = (prediction: string) =>\n  prediction\n    .replace(/Reflection:[\\s\\S]*?(?=Action_Summary:|Action:|$)/g, '')\n    .trim();\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getUiTarsPlanningPrompt","preferredLanguage","getPreferredLanguage","getSummary","prediction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACJO,SAASI;IACd,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;MAuBJ,EAAED,kBAAkB;;;;AAI1B,CAAC;AACD;AAEO,MAAME,aAAa,CAACC,aACzBA,WACG,OAAO,CAAC,qDAAqD,IAC7D,IAAI"}
+{"version":3,"file":"ai-model/prompt/ui-tars-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/ui-tars-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getPreferredLanguage } from '@aiscene/shared/env';\n\nexport function getUiTarsPlanningPrompt(): string {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. \n\n## Output Format\n\\`\\`\\`\nThought: ...\nAction: ...\n\\`\\`\\`\n\n## Action Space\n\nclick(start_box='[x1, y1, x2, y2]')\nleft_double(start_box='[x1, y1, x2, y2]')\nright_single(start_box='[x1, y1, x2, y2]')\ndrag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')\nhotkey(key='')\ntype(content='xxx') # Use escape characters \\\\', \\\\\\\", and \\\\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\\\n at the end of content. \nscroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')\nwait() #Sleep for 5s and take a screenshot to check for any changes.\nfinished(content='xxx') # Use escape characters \\\\', \\\\\", and \\\\n in content part to ensure we can parse the content in normal python string format.\n\n\n## Note\n- Use ${preferredLanguage} in \\`Thought\\` part.\n- Write a small plan and finally summarize your next action (with its target element) in one sentence in \\`Thought\\` part.\n\n## User Instruction\n`;\n}\n\nexport const getSummary = (prediction: string) =>\n  prediction\n    .replace(/Reflection:[\\s\\S]*?(?=Action_Summary:|Action:|$)/g, '')\n    .trim();\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getUiTarsPlanningPrompt","preferredLanguage","getPreferredLanguage","getSummary","prediction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACJO,SAASI;IACd,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;MAuBJ,EAAED,kBAAkB;;;;AAI1B,CAAC;AACD;AAEO,MAAME,aAAa,CAACC,aACzBA,WACG,OAAO,CAAC,qDAAqD,IAC7D,IAAI"}

package/dist/lib/ai-model/prompt/yaml-generator.js

@@ -35,7 +35,7 @@ __webpack_require__.d(__webpack_exports__, {
     prepareEventSummary: ()=>prepareEventSummary,
     generateYamlTest: ()=>generateYamlTest
 });
-const constants_namespaceObject = require("@midscene/shared/constants");
+const constants_namespaceObject = require("@aiscene/shared/constants");
 const external_index_js_namespaceObject = require("../index.js");
 const getScreenshotsForLLM = (events, maxScreenshots = 1)=>{
     const eventsWithScreenshots = events.filter((event)=>event.screenshotBefore || event.screenshotAfter || event.screenshotWithBox);

package/dist/lib/ai-model/prompt/yaml-generator.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/yaml-generator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/yaml-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type ChatCompletionMessageParam,\n  callAI,\n  callAIWithStringResponse,\n} from '../index';\n\n// Common interfaces for test generation (shared between YAML and Playwright)\nexport interface EventCounts {\n  navigation: number;\n  click: number;\n  input: number;\n  scroll: number;\n  total: number;\n}\n\nexport interface InputDescription {\n  description: string;\n  value: string;\n}\n\nexport interface ProcessedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n}\n\nexport interface EventSummary {\n  testName: string;\n  startUrl: string;\n  eventCounts: EventCounts;\n  urls: string[];\n  clickDescriptions: string[];\n  inputDescriptions: InputDescription[];\n  events: ProcessedEvent[];\n}\n\n// Common ChromeRecordedEvent interface\nexport interface ChromeRecordedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n  screenshotBefore?: string;\n  screenshotAfter?: string;\n  screenshotWithBox?: string;\n}\n\nexport interface YamlGenerationOptions {\n  testName?: string;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  /** Language for human-readable YAML content (e.g. 'English', 'Chinese'). Keys and API names are kept as-is. */\n  language?: string;\n}\n\nexport interface FilteredEvents {\n  navigationEvents: ChromeRecordedEvent[];\n  clickEvents: ChromeRecordedEvent[];\n  inputEvents: ChromeRecordedEvent[];\n  scrollEvents: ChromeRecordedEvent[];\n}\n\n// Common utility functions (shared between YAML and Playwright generators)\n\n/**\n * Get screenshots from events for LLM context\n */\nexport const getScreenshotsForLLM = (\n  events: ChromeRecordedEvent[],\n  maxScreenshots = 1,\n): string[] => {\n  // Find events with screenshots, prioritizing navigation and click events\n  const eventsWithScreenshots = events.filter(\n    (event) =>\n      event.screenshotBefore ||\n      event.screenshotAfter ||\n      event.screenshotWithBox,\n  );\n\n  // Sort them by priority (navigation first, then clicks, then others)\n  const sortedEvents = [...eventsWithScreenshots].sort((a, b) => {\n    if (a.type === 'navigation' && b.type !== 'navigation') return -1;\n    if (a.type !== 'navigation' && b.type === 'navigation') return 1;\n    if (a.type === 'click' && b.type !== 'click') return -1;\n    if (a.type !== 'click' && b.type === 'click') return 1;\n    return 0;\n  });\n\n  // Extract up to maxScreenshots screenshots\n  const screenshots: string[] = [];\n  for (const event of sortedEvents) {\n    // Prefer the most informative screenshot\n    const screenshot =\n      event.screenshotWithBox ||\n      event.screenshotAfter ||\n      event.screenshotBefore;\n    if (screenshot && !screenshots.includes(screenshot)) {\n      screenshots.push(screenshot);\n      if (screenshots.length >= maxScreenshots) break;\n    }\n  }\n\n  return screenshots;\n};\n\n/**\n * Filter events by type for easier processing\n */\nexport const filterEventsByType = (\n  events: ChromeRecordedEvent[],\n): FilteredEvents => {\n  return {\n    navigationEvents: events.filter((event) => event.type === 'navigation'),\n    clickEvents: events.filter((event) => event.type === 'click'),\n    inputEvents: events.filter((event) => event.type === 'input'),\n    scrollEvents: events.filter((event) => event.type === 'scroll'),\n  };\n};\n\n/**\n * Create event counts summary\n */\nexport const createEventCounts = (\n  filteredEvents: FilteredEvents,\n  totalEvents: number,\n): EventCounts => {\n  return {\n    navigation: filteredEvents.navigationEvents.length,\n    click: filteredEvents.clickEvents.length,\n    input: filteredEvents.inputEvents.length,\n    scroll: filteredEvents.scrollEvents.length,\n    total: totalEvents,\n  };\n};\n\n/**\n * Extract input descriptions from input events\n */\nexport const extractInputDescriptions = (\n  inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n  return inputEvents\n    .map((event) => ({\n      description: event.elementDescription || '',\n      value: event.value || '',\n    }))\n    .filter((item) => item.description && item.value);\n};\n\n/**\n * Process events for LLM consumption\n */\nexport const processEventsForLLM = (\n  events: ChromeRecordedEvent[],\n): ProcessedEvent[] => {\n  return events.map((event) => ({\n    type: event.type,\n    timestamp: event.timestamp,\n    url: event.url,\n    title: event.title,\n    elementDescription: event.elementDescription,\n    value: event.value,\n    pageInfo: event.pageInfo,\n    elementRect: event.elementRect,\n  }));\n};\n\n/**\n * Prepare comprehensive event summary for LLM\n */\nexport const prepareEventSummary = (\n  events: ChromeRecordedEvent[],\n  options: { testName?: string; maxScreenshots?: number } = {},\n): EventSummary => {\n  const filteredEvents = filterEventsByType(events);\n  const eventCounts = createEventCounts(filteredEvents, events.length);\n\n  // Extract useful information from events\n  const startUrl =\n    filteredEvents.navigationEvents.length > 0\n      ? filteredEvents.navigationEvents[0].url || ''\n      : '';\n\n  const clickDescriptions = filteredEvents.clickEvents\n    .map((event) => event.elementDescription)\n    .filter((desc): desc is string => Boolean(desc))\n    .slice(0, 10);\n\n  const inputDescriptions = extractInputDescriptions(\n    filteredEvents.inputEvents,\n  ).slice(0, 10);\n\n  const urls = filteredEvents.navigationEvents\n    .map((e) => e.url)\n    .filter((url): url is string => Boolean(url))\n    .slice(0, 5);\n\n  const processedEvents = processEventsForLLM(events);\n\n  return {\n    testName: options.testName || 'Automated test from recorded events',\n    startUrl,\n    eventCounts,\n    urls,\n    clickDescriptions,\n    inputDescriptions,\n    events: processedEvents,\n  };\n};\n\n/**\n * Create message content for LLM with optional screenshots\n */\nexport const createMessageContent = (\n  promptText: string,\n  screenshots: string[] = [],\n  includeScreenshots = true,\n) => {\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  // Add screenshots if available and requested\n  if (includeScreenshots && screenshots.length > 0) {\n    messageContent.unshift({\n      type: 'text',\n      text: 'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    screenshots.forEach((screenshot) => {\n      messageContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      });\n    });\n  }\n\n  return messageContent;\n};\n\nconst getYamlLanguageInstruction = (language?: string) => {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable YAML content in ${normalizedLanguage}.\n- Keep YAML keys, field names, and Midscene API names unchanged.`;\n};\n\nconst createYamlPrompt = ({\n  yamlSummary,\n  screenshots,\n  language,\n}: {\n  yamlSummary: EventSummary & { includeTimestamps: boolean };\n  screenshots: string[];\n  language?: string;\n}): ChatCompletionMessageParam[] => {\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n    },\n    {\n      role: 'user',\n      content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes${getYamlLanguageInstruction(language)}\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n    },\n  ];\n\n  if (screenshots.length > 0) {\n    prompt.push({\n      role: 'user',\n      content:\n        'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    prompt.push({\n      role: 'user',\n      content: screenshots.map((screenshot) => ({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      })),\n    });\n  }\n\n  return prompt;\n};\n\n/**\n * Validate events before processing\n */\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n  if (!events.length) {\n    throw new Error('No events provided for test generation');\n  }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    const prompt = createYamlPrompt({\n      yamlSummary,\n      screenshots,\n      language: options.language,\n    });\n\n    const response = await callAIWithStringResponse(prompt, modelConfig);\n\n    if (response?.content && typeof response.content === 'string') {\n      return response.content;\n    }\n\n    throw new Error('Failed to generate YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    const prompt = createYamlPrompt({\n      yamlSummary,\n      screenshots,\n      language: options.language,\n    });\n\n    if (options.stream && options.onChunk) {\n      // Use streaming\n      return await callAI(prompt, modelConfig, {\n        stream: true,\n        onChunk: options.onChunk,\n      });\n    } else {\n      // Fallback to non-streaming\n      const response = await callAIWithStringResponse(prompt, modelConfig);\n\n      if (response?.content && typeof response.content === 'string') {\n        return {\n          content: response.content,\n          usage: response.usage,\n          isStreamed: false,\n        };\n      }\n\n      throw new Error('Failed to generate YAML test configuration');\n    }\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getScreenshotsForLLM","events","maxScreenshots","eventsWithScreenshots","event","sortedEvents","a","b","screenshots","screenshot","filterEventsByType","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","item","processEventsForLLM","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","Boolean","inputDescriptions","urls","e","url","processedEvents","createMessageContent","promptText","includeScreenshots","messageContent","getYamlLanguageInstruction","language","normalizedLanguage","createYamlPrompt","yamlSummary","prompt","YAML_EXAMPLE_CODE","JSON","validateEvents","Error","generateYamlTest","modelConfig","summary","response","callAIWithStringResponse","error","generateYamlTestStream","callAI"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;AC6EO,MAAMI,uBAAuB,CAClCC,QACAC,iBAAiB,CAAC;IAGlB,MAAMC,wBAAwBF,OAAO,MAAM,CACzC,CAACG,QACCA,MAAM,gBAAgB,IACtBA,MAAM,eAAe,IACrBA,MAAM,iBAAiB;IAI3B,MAAMC,eAAe;WAAIF;KAAsB,CAAC,IAAI,CAAC,CAACG,GAAGC;QACvD,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,OAAO;IACT;IAGA,MAAMC,cAAwB,EAAE;IAChC,KAAK,MAAMJ,SAASC,aAAc;QAEhC,MAAMI,aACJL,MAAM,iBAAiB,IACvBA,MAAM,eAAe,IACrBA,MAAM,gBAAgB;QACxB,IAAIK,cAAc,CAACD,YAAY,QAAQ,CAACC,aAAa;YACnDD,YAAY,IAAI,CAACC;YACjB,IAAID,YAAY,MAAM,IAAIN,gBAAgB;QAC5C;IACF;IAEA,OAAOM;AACT;AAKO,MAAME,qBAAqB,CAChCT,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAMK,MAAMO,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAMK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACX,QAAW;YACf,aAAaA,MAAM,kBAAkB,IAAI;YACzC,OAAOA,MAAM,KAAK,IAAI;QACxB,IACC,MAAM,CAAC,CAACY,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAM7C,MAAMC,sBAAsB,CACjChB,SAEOA,OAAO,GAAG,CAAC,CAACG,QAAW;YAC5B,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClB,oBAAoBA,MAAM,kBAAkB;YAC5C,OAAOA,MAAM,KAAK;YAClB,UAAUA,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;QAChC;AAMK,MAAMc,sBAAsB,CACjCjB,QACAkB,UAA0D,CAAC,CAAC;IAE5D,MAAMP,iBAAiBF,mBAAmBT;IAC1C,MAAMmB,cAAcT,kBAAkBC,gBAAgBX,OAAO,MAAM;IAGnE,MAAMoB,WACJT,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAMU,oBAAoBV,eAAe,WAAW,CACjD,GAAG,CAAC,CAACR,QAAUA,MAAM,kBAAkB,EACvC,MAAM,CAAC,CAACmB,OAAyBC,QAAQD,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAME,oBAAoBX,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAMc,OAAOd,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAACe,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBJ,QAAQI,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkBZ,oBAAoBhB;IAE5C,OAAO;QACL,UAAUkB,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAM;QACAJ;QACAG;QACA,QAAQI;IACV;AACF;AAKO,MAAMC,uBAAuB,CAClCC,YACAvB,cAAwB,EAAE,EAC1BwB,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMF;QACR;KACD;IAGD,IAAIC,sBAAsBxB,YAAY,MAAM,GAAG,GAAG;QAChDyB,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAzB,YAAY,OAAO,CAAC,CAACC;YACnBwB,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKxB;gBACP;YACF;QACF;IACF;IAEA,OAAOwB;AACT;AAEA,MAAMC,6BAA6B,CAACC;IAClC,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;2CAEiC,EAAEA,mBAAmB;gEACA,CAAC;AACjE;AAEA,MAAMC,mBAAmB,CAAC,EACxBC,WAAW,EACX9B,WAAW,EACX2B,QAAQ,EAKT;IACC,MAAMI,SAAuC;QAC3C;YACE,MAAM;YACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;QAC7I;QACA;YACE,MAAM;YACN,SAAS,CAAC;;;AAGhB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;0CAOG,EAAEJ,2BAA2BC,UAAU;;8JAE6E,CAAC;QAC3J;KACD;IAED,IAAI3B,YAAY,MAAM,GAAG,GAAG;QAC1B+B,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SACE;QACJ;QAEAA,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SAAS/B,YAAY,GAAG,CAAC,CAACC,aAAgB;oBACxC,MAAM;oBACN,WAAW;wBACT,KAAKA;oBACP;gBACF;QACF;IACF;IAEA,OAAO8B;AACT;AAKO,MAAMG,iBAAiB,CAACzC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAI0C,MAAM;AAEpB;AAOO,MAAMC,mBAAmB,OAC9B3C,QACAkB,SACA0B;IAEA,IAAI;QAEFH,eAAezC;QAGf,MAAM6C,UAAU5B,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMmB,cAAc;YAClB,GAAGQ,OAAO;YACV,mBAAmB3B,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAG5B,MAAMoB,SAASF,iBAAiB;YAC9BC;YACA9B;YACA,UAAUW,QAAQ,QAAQ;QAC5B;QAEA,MAAM4B,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBT,QAAQM;QAExD,IAAIE,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;QAGzB,MAAM,IAAIJ,MAAM;IAClB,EAAE,OAAOM,OAAO;QACd,MAAM,IAAIN,MAAM,CAAC,8BAA8B,EAAEM,OAAO;IAC1D;AACF;AAKO,MAAMC,yBAAyB,OACpCjD,QACAkB,SACA0B;IAEA,IAAI;QAEFH,eAAezC;QAGf,MAAM6C,UAAU5B,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMmB,cAAc;YAClB,GAAGQ,OAAO;YACV,mBAAmB3B,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAG5B,MAAMoB,SAASF,iBAAiB;YAC9BC;YACA9B;YACA,UAAUW,QAAQ,QAAQ;QAC5B;QAEA,IAAIA,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMgC,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOZ,QAAQM,aAAa;YACvC,QAAQ;YACR,SAAS1B,QAAQ,OAAO;QAC1B;QACK;YAEL,MAAM4B,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBT,QAAQM;YAExD,IAAIE,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;gBACL,SAASA,SAAS,OAAO;gBACzB,OAAOA,SAAS,KAAK;gBACrB,YAAY;YACd;YAGF,MAAM,IAAIJ,MAAM;QAClB;IACF,EAAE,OAAOM,OAAO;QACd,MAAM,IAAIN,MAAM,CAAC,8BAA8B,EAAEM,OAAO;IAC1D;AACF"}
+{"version":3,"file":"ai-model/prompt/yaml-generator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/yaml-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@aiscene/shared/constants';\nimport type { IModelConfig } from '@aiscene/shared/env';\nimport {\n  type ChatCompletionMessageParam,\n  callAI,\n  callAIWithStringResponse,\n} from '../index';\n\n// Common interfaces for test generation (shared between YAML and Playwright)\nexport interface EventCounts {\n  navigation: number;\n  click: number;\n  input: number;\n  scroll: number;\n  total: number;\n}\n\nexport interface InputDescription {\n  description: string;\n  value: string;\n}\n\nexport interface ProcessedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n}\n\nexport interface EventSummary {\n  testName: string;\n  startUrl: string;\n  eventCounts: EventCounts;\n  urls: string[];\n  clickDescriptions: string[];\n  inputDescriptions: InputDescription[];\n  events: ProcessedEvent[];\n}\n\n// Common ChromeRecordedEvent interface\nexport interface ChromeRecordedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n  screenshotBefore?: string;\n  screenshotAfter?: string;\n  screenshotWithBox?: string;\n}\n\nexport interface YamlGenerationOptions {\n  testName?: string;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  /** Language for human-readable YAML content (e.g. 'English', 'Chinese'). Keys and API names are kept as-is. */\n  language?: string;\n}\n\nexport interface FilteredEvents {\n  navigationEvents: ChromeRecordedEvent[];\n  clickEvents: ChromeRecordedEvent[];\n  inputEvents: ChromeRecordedEvent[];\n  scrollEvents: ChromeRecordedEvent[];\n}\n\n// Common utility functions (shared between YAML and Playwright generators)\n\n/**\n * Get screenshots from events for LLM context\n */\nexport const getScreenshotsForLLM = (\n  events: ChromeRecordedEvent[],\n  maxScreenshots = 1,\n): string[] => {\n  // Find events with screenshots, prioritizing navigation and click events\n  const eventsWithScreenshots = events.filter(\n    (event) =>\n      event.screenshotBefore ||\n      event.screenshotAfter ||\n      event.screenshotWithBox,\n  );\n\n  // Sort them by priority (navigation first, then clicks, then others)\n  const sortedEvents = [...eventsWithScreenshots].sort((a, b) => {\n    if (a.type === 'navigation' && b.type !== 'navigation') return -1;\n    if (a.type !== 'navigation' && b.type === 'navigation') return 1;\n    if (a.type === 'click' && b.type !== 'click') return -1;\n    if (a.type !== 'click' && b.type === 'click') return 1;\n    return 0;\n  });\n\n  // Extract up to maxScreenshots screenshots\n  const screenshots: string[] = [];\n  for (const event of sortedEvents) {\n    // Prefer the most informative screenshot\n    const screenshot =\n      event.screenshotWithBox ||\n      event.screenshotAfter ||\n      event.screenshotBefore;\n    if (screenshot && !screenshots.includes(screenshot)) {\n      screenshots.push(screenshot);\n      if (screenshots.length >= maxScreenshots) break;\n    }\n  }\n\n  return screenshots;\n};\n\n/**\n * Filter events by type for easier processing\n */\nexport const filterEventsByType = (\n  events: ChromeRecordedEvent[],\n): FilteredEvents => {\n  return {\n    navigationEvents: events.filter((event) => event.type === 'navigation'),\n    clickEvents: events.filter((event) => event.type === 'click'),\n    inputEvents: events.filter((event) => event.type === 'input'),\n    scrollEvents: events.filter((event) => event.type === 'scroll'),\n  };\n};\n\n/**\n * Create event counts summary\n */\nexport const createEventCounts = (\n  filteredEvents: FilteredEvents,\n  totalEvents: number,\n): EventCounts => {\n  return {\n    navigation: filteredEvents.navigationEvents.length,\n    click: filteredEvents.clickEvents.length,\n    input: filteredEvents.inputEvents.length,\n    scroll: filteredEvents.scrollEvents.length,\n    total: totalEvents,\n  };\n};\n\n/**\n * Extract input descriptions from input events\n */\nexport const extractInputDescriptions = (\n  inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n  return inputEvents\n    .map((event) => ({\n      description: event.elementDescription || '',\n      value: event.value || '',\n    }))\n    .filter((item) => item.description && item.value);\n};\n\n/**\n * Process events for LLM consumption\n */\nexport const processEventsForLLM = (\n  events: ChromeRecordedEvent[],\n): ProcessedEvent[] => {\n  return events.map((event) => ({\n    type: event.type,\n    timestamp: event.timestamp,\n    url: event.url,\n    title: event.title,\n    elementDescription: event.elementDescription,\n    value: event.value,\n    pageInfo: event.pageInfo,\n    elementRect: event.elementRect,\n  }));\n};\n\n/**\n * Prepare comprehensive event summary for LLM\n */\nexport const prepareEventSummary = (\n  events: ChromeRecordedEvent[],\n  options: { testName?: string; maxScreenshots?: number } = {},\n): EventSummary => {\n  const filteredEvents = filterEventsByType(events);\n  const eventCounts = createEventCounts(filteredEvents, events.length);\n\n  // Extract useful information from events\n  const startUrl =\n    filteredEvents.navigationEvents.length > 0\n      ? filteredEvents.navigationEvents[0].url || ''\n      : '';\n\n  const clickDescriptions = filteredEvents.clickEvents\n    .map((event) => event.elementDescription)\n    .filter((desc): desc is string => Boolean(desc))\n    .slice(0, 10);\n\n  const inputDescriptions = extractInputDescriptions(\n    filteredEvents.inputEvents,\n  ).slice(0, 10);\n\n  const urls = filteredEvents.navigationEvents\n    .map((e) => e.url)\n    .filter((url): url is string => Boolean(url))\n    .slice(0, 5);\n\n  const processedEvents = processEventsForLLM(events);\n\n  return {\n    testName: options.testName || 'Automated test from recorded events',\n    startUrl,\n    eventCounts,\n    urls,\n    clickDescriptions,\n    inputDescriptions,\n    events: processedEvents,\n  };\n};\n\n/**\n * Create message content for LLM with optional screenshots\n */\nexport const createMessageContent = (\n  promptText: string,\n  screenshots: string[] = [],\n  includeScreenshots = true,\n) => {\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  // Add screenshots if available and requested\n  if (includeScreenshots && screenshots.length > 0) {\n    messageContent.unshift({\n      type: 'text',\n      text: 'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    screenshots.forEach((screenshot) => {\n      messageContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      });\n    });\n  }\n\n  return messageContent;\n};\n\nconst getYamlLanguageInstruction = (language?: string) => {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable YAML content in ${normalizedLanguage}.\n- Keep YAML keys, field names, and Midscene API names unchanged.`;\n};\n\nconst createYamlPrompt = ({\n  yamlSummary,\n  screenshots,\n  language,\n}: {\n  yamlSummary: EventSummary & { includeTimestamps: boolean };\n  screenshots: string[];\n  language?: string;\n}): ChatCompletionMessageParam[] => {\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n    },\n    {\n      role: 'user',\n      content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes${getYamlLanguageInstruction(language)}\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n    },\n  ];\n\n  if (screenshots.length > 0) {\n    prompt.push({\n      role: 'user',\n      content:\n        'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    prompt.push({\n      role: 'user',\n      content: screenshots.map((screenshot) => ({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      })),\n    });\n  }\n\n  return prompt;\n};\n\n/**\n * Validate events before processing\n */\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n  if (!events.length) {\n    throw new Error('No events provided for test generation');\n  }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    const prompt = createYamlPrompt({\n      yamlSummary,\n      screenshots,\n      language: options.language,\n    });\n\n    const response = await callAIWithStringResponse(prompt, modelConfig);\n\n    if (response?.content && typeof response.content === 'string') {\n      return response.content;\n    }\n\n    throw new Error('Failed to generate YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    const prompt = createYamlPrompt({\n      yamlSummary,\n      screenshots,\n      language: options.language,\n    });\n\n    if (options.stream && options.onChunk) {\n      // Use streaming\n      return await callAI(prompt, modelConfig, {\n        stream: true,\n        onChunk: options.onChunk,\n      });\n    } else {\n      // Fallback to non-streaming\n      const response = await callAIWithStringResponse(prompt, modelConfig);\n\n      if (response?.content && typeof response.content === 'string') {\n        return {\n          content: response.content,\n          usage: response.usage,\n          isStreamed: false,\n        };\n      }\n\n      throw new Error('Failed to generate YAML test configuration');\n    }\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getScreenshotsForLLM","events","maxScreenshots","eventsWithScreenshots","event","sortedEvents","a","b","screenshots","screenshot","filterEventsByType","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","item","processEventsForLLM","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","Boolean","inputDescriptions","urls","e","url","processedEvents","createMessageContent","promptText","includeScreenshots","messageContent","getYamlLanguageInstruction","language","normalizedLanguage","createYamlPrompt","yamlSummary","prompt","YAML_EXAMPLE_CODE","JSON","validateEvents","Error","generateYamlTest","modelConfig","summary","response","callAIWithStringResponse","error","generateYamlTestStream","callAI"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;AC6EO,MAAMI,uBAAuB,CAClCC,QACAC,iBAAiB,CAAC;IAGlB,MAAMC,wBAAwBF,OAAO,MAAM,CACzC,CAACG,QACCA,MAAM,gBAAgB,IACtBA,MAAM,eAAe,IACrBA,MAAM,iBAAiB;IAI3B,MAAMC,eAAe;WAAIF;KAAsB,CAAC,IAAI,CAAC,CAACG,GAAGC;QACvD,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,OAAO;IACT;IAGA,MAAMC,cAAwB,EAAE;IAChC,KAAK,MAAMJ,SAASC,aAAc;QAEhC,MAAMI,aACJL,MAAM,iBAAiB,IACvBA,MAAM,eAAe,IACrBA,MAAM,gBAAgB;QACxB,IAAIK,cAAc,CAACD,YAAY,QAAQ,CAACC,aAAa;YACnDD,YAAY,IAAI,CAACC;YACjB,IAAID,YAAY,MAAM,IAAIN,gBAAgB;QAC5C;IACF;IAEA,OAAOM;AACT;AAKO,MAAME,qBAAqB,CAChCT,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAMK,MAAMO,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAMK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACX,QAAW;YACf,aAAaA,MAAM,kBAAkB,IAAI;YACzC,OAAOA,MAAM,KAAK,IAAI;QACxB,IACC,MAAM,CAAC,CAACY,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAM7C,MAAMC,sBAAsB,CACjChB,SAEOA,OAAO,GAAG,CAAC,CAACG,QAAW;YAC5B,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClB,oBAAoBA,MAAM,kBAAkB;YAC5C,OAAOA,MAAM,KAAK;YAClB,UAAUA,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;QAChC;AAMK,MAAMc,sBAAsB,CACjCjB,QACAkB,UAA0D,CAAC,CAAC;IAE5D,MAAMP,iBAAiBF,mBAAmBT;IAC1C,MAAMmB,cAAcT,kBAAkBC,gBAAgBX,OAAO,MAAM;IAGnE,MAAMoB,WACJT,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAMU,oBAAoBV,eAAe,WAAW,CACjD,GAAG,CAAC,CAACR,QAAUA,MAAM,kBAAkB,EACvC,MAAM,CAAC,CAACmB,OAAyBC,QAAQD,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAME,oBAAoBX,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAMc,OAAOd,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAACe,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBJ,QAAQI,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkBZ,oBAAoBhB;IAE5C,OAAO;QACL,UAAUkB,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAM;QACAJ;QACAG;QACA,QAAQI;IACV;AACF;AAKO,MAAMC,uBAAuB,CAClCC,YACAvB,cAAwB,EAAE,EAC1BwB,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMF;QACR;KACD;IAGD,IAAIC,sBAAsBxB,YAAY,MAAM,GAAG,GAAG;QAChDyB,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAzB,YAAY,OAAO,CAAC,CAACC;YACnBwB,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKxB;gBACP;YACF;QACF;IACF;IAEA,OAAOwB;AACT;AAEA,MAAMC,6BAA6B,CAACC;IAClC,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;2CAEiC,EAAEA,mBAAmB;gEACA,CAAC;AACjE;AAEA,MAAMC,mBAAmB,CAAC,EACxBC,WAAW,EACX9B,WAAW,EACX2B,QAAQ,EAKT;IACC,MAAMI,SAAuC;QAC3C;YACE,MAAM;YACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;QAC7I;QACA;YACE,MAAM;YACN,SAAS,CAAC;;;AAGhB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;0CAOG,EAAEJ,2BAA2BC,UAAU;;8JAE6E,CAAC;QAC3J;KACD;IAED,IAAI3B,YAAY,MAAM,GAAG,GAAG;QAC1B+B,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SACE;QACJ;QAEAA,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SAAS/B,YAAY,GAAG,CAAC,CAACC,aAAgB;oBACxC,MAAM;oBACN,WAAW;wBACT,KAAKA;oBACP;gBACF;QACF;IACF;IAEA,OAAO8B;AACT;AAKO,MAAMG,iBAAiB,CAACzC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAI0C,MAAM;AAEpB;AAOO,MAAMC,mBAAmB,OAC9B3C,QACAkB,SACA0B;IAEA,IAAI;QAEFH,eAAezC;QAGf,MAAM6C,UAAU5B,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMmB,cAAc;YAClB,GAAGQ,OAAO;YACV,mBAAmB3B,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAG5B,MAAMoB,SAASF,iBAAiB;YAC9BC;YACA9B;YACA,UAAUW,QAAQ,QAAQ;QAC5B;QAEA,MAAM4B,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBT,QAAQM;QAExD,IAAIE,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;QAGzB,MAAM,IAAIJ,MAAM;IAClB,EAAE,OAAOM,OAAO;QACd,MAAM,IAAIN,MAAM,CAAC,8BAA8B,EAAEM,OAAO;IAC1D;AACF;AAKO,MAAMC,yBAAyB,OACpCjD,QACAkB,SACA0B;IAEA,IAAI;QAEFH,eAAezC;QAGf,MAAM6C,UAAU5B,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMmB,cAAc;YAClB,GAAGQ,OAAO;YACV,mBAAmB3B,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAG5B,MAAMoB,SAASF,iBAAiB;YAC9BC;YACA9B;YACA,UAAUW,QAAQ,QAAQ;QAC5B;QAEA,IAAIA,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMgC,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOZ,QAAQM,aAAa;YACvC,QAAQ;YACR,SAAS1B,QAAQ,OAAO;QAC1B;QACK;YAEL,MAAM4B,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBT,QAAQM;YAExD,IAAIE,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;gBACL,SAASA,SAAS,OAAO;gBACzB,OAAOA,SAAS,KAAK;gBACrB,YAAY;YACd;YAGF,MAAM,IAAIJ,MAAM;QAClB;IACF,EAAE,OAAOM,OAAO;QACd,MAAM,IAAIN,MAAM,CAAC,8BAA8B,EAAEM,OAAO;IAC1D;AACF"}

package/dist/lib/ai-model/service-caller/codex-app-server.js

@@ -31,8 +31,8 @@ __webpack_require__.d(__webpack_exports__, {
     resolveCodexReasoningEffort: ()=>resolveCodexReasoningEffort,
     buildCodexTurnPayloadFromMessages: ()=>buildCodexTurnPayloadFromMessages
 });
-const logger_namespaceObject = require("@midscene/shared/logger");
-const utils_namespaceObject = require("@midscene/shared/utils");
+const logger_namespaceObject = require("@aiscene/shared/logger");
+const utils_namespaceObject = require("@aiscene/shared/utils");
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,