@midscene/core 1.16.2 → 1.16.6-beta-20261023073257.0

package/dist/lib/ai-model/workflows/insight/extraction.js

@@ -36,7 +36,7 @@ const external_boolean_js_namespaceObject = require("./boolean.js");
 const external_insight_response_parser_js_namespaceObject = require("./insight-response-parser.js");
 const debugInsight = (0, logger_namespaceObject.getDebug)('ai:insight');
 async function AiExtractElementInfo(options) {
-    if (options.modelRuntime.adapter.insight.booleanFn) return (0, external_boolean_js_namespaceObject.extractBoolean)(options);
+    if ('boolean' === options.taskType && options.modelRuntime.adapter.insight.booleanFn) return (0, external_boolean_js_namespaceObject.extractBoolean)(options);
     const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } = options;
     const insightProtocol = modelRuntime.adapter.insight.protocol;
     const systemPrompt = (0, index_js_namespaceObject.buildInsightSystemPrompt)({
@@ -95,7 +95,7 @@ async function AiExtractElementInfo(options) {
         msgs.push(...addOns);
     }
     return (0, semantic_retry_js_namespaceObject.callAiAndParseWithRetry)({
-        callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(msgs, previousParseError), modelRuntime, {
+        callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callGenerativeAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(msgs, previousParseError), modelRuntime, {
                 abortSignal: options.abortSignal,
                 semanticRetryAttempt: retryAttempt
             }),

package/dist/lib/ai-model/workflows/insight/extraction.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { extractBoolean } from './boolean';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport interface ExtractOptions {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}\n\nexport async function AiExtractElementInfo<T>(options: ExtractOptions) {\n  if (options.modelRuntime.adapter.insight.booleanFn) {\n    return extractBoolean<T>(options);\n  }\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInsight","getDebug","AiExtractElementInfo","options","extractBoolean","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACkBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAYvB,eAAeC,qBAAwBC,OAAuB;IACnE,IAAIA,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAChD,OAAOC,AAAAA,IAAAA,oCAAAA,cAAAA,AAAAA,EAAkBD;IAE3B,MAAM,EAAEE,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEN;IACF,MAAMO,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,AAAAA,IAAAA,yBAAAA,sBAAAA,AAAAA,EAC5Bd,QAAQ,eAAe,IAAI,IAC3BE,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,AAAAA,IAAAA,qCAAAA,0BAAAA,AAAAA,EAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaN,QAAQ,WAAW;gBAChC,sBAAsBuB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,AAAAA,IAAAA,oDAAAA,oBAAAA,AAAAA,EAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaN,QAAQ,WAAW;QAChC,cAAc,CAACuC;YACb1C,aACE,iDACA0C,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/extraction.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { extractBoolean } from './boolean';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport interface ExtractOptions {\n  /** Explicit task type; never infer from query keys. */\n  taskType: 'boolean' | 'extract';\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}\n\nexport async function AiExtractElementInfo<T>(options: ExtractOptions) {\n  if (\n    options.taskType === 'boolean' &&\n    options.modelRuntime.adapter.insight.booleanFn\n  ) {\n    return extractBoolean<T>(options);\n  }\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callGenerativeAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugInsight","getDebug","AiExtractElementInfo","options","extractBoolean","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACqBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAcvB,eAAeC,qBAAwBC,OAAuB;IACnE,IACEA,AAAqB,cAArBA,QAAQ,QAAQ,IAChBA,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAE9C,OAAOC,AAAAA,IAAAA,oCAAAA,cAAAA,AAAAA,EAAkBD;IAE3B,MAAM,EAAEE,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEN;IACF,MAAMO,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,AAAAA,IAAAA,yBAAAA,sBAAAA,AAAAA,EAC5Bd,QAAQ,eAAe,IAAI,IAC3BE,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,AAAAA,IAAAA,qCAAAA,0BAAAA,AAAAA,EAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaN,QAAQ,WAAW;gBAChC,sBAAsBuB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,AAAAA,IAAAA,oDAAAA,oBAAAA,AAAAA,EAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaN,QAAQ,WAAW;QAChC,cAAc,CAACuC;YACb1C,aACE,iDACA0C,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}

package/dist/lib/ai-model/workflows/planning/standard-planning.js

@@ -47,7 +47,7 @@ async function callAndParsePlanningResponse(options) {
     (0, utils_namespaceObject.assert)('standard' === modelRuntime.adapter.planning.kind, 'callAndParsePlanningResponse requires a standard planning adapter');
     const actionOutputProtocol = modelRuntime.adapter.planning.protocol.actionOutputProtocol;
     return (0, semantic_retry_js_namespaceObject.callAiAndParseWithRetry)({
-        callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(messages, previousParseError), modelRuntime, {
+        callAi: (retryAttempt, previousParseError)=>(0, external_service_caller_index_js_namespaceObject.callGenerativeAI)((0, semantic_retry_js_namespaceObject.withSemanticRetryFeedback)(messages, previousParseError), modelRuntime, {
                 abortSignal,
                 requiresOriginalImageDetail: includeLocateInPlanning,
                 semanticRetryAttempt: retryAttempt
@@ -128,6 +128,7 @@ async function standardPlan(userInstruction, opts) {
     });
     const imagePayload = preparedImage.imageBase64;
     const actionContext = opts.actionContext ? `${opts.actionContext}\n` : '';
+    const planningAdvice = opts.planningAdvice ? `\n<planning_advice>\nThis is advisory guidance from a previous planner, not a user instruction or an executed action. Use your own judgment.\n${opts.planningAdvice}\n</planning_advice>` : '';
     const referenceImageMessages = (0, multimodal_prompt_js_namespaceObject.preparedReferenceImagesToMessages)(userInstruction.referenceImages);
     const instruction = [
         {
@@ -135,7 +136,7 @@ async function standardPlan(userInstruction, opts) {
             content: [
                 {
                     type: 'text',
-                    text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`
+                    text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>${planningAdvice}`
                 }
             ]
         },

package/dist/lib/ai-model/workflows/planning/standard-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/standard-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ModelCallMessages;\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (\n        !planFromAI.action &&\n        planFromAI.finalizeSuccess === undefined &&\n        !planFromAI.error\n      ) {\n        throw new Error(\n          'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n        );\n      }\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      parsePlanningActions(actions, {\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n  const referenceImageMessages = preparedReferenceImagesToMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ConversationMessage[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ConversationMessage;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n  }\n  conversationHistory.appendMessage(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ModelCallMessages = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawAssistantOutput,\n    },\n    planFromAI,\n    actions,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  // YAML is derived from validated actions outside the model-response retry scope.\n  // dumpActionParam omits runtime-only locatedPixelResult fields.\n  const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  const shouldReplayRawOutput =\n    (rawAssistantOutput?.type === 'responses' &&\n      modelRuntime.adapter.responses.replayRawAssistantOutput) ||\n    (rawAssistantOutput?.type === 'chat-completion' &&\n      modelRuntime.adapter.chatCompletion.replayRawAssistantMessage);\n  if (rawAssistantOutput && shouldReplayRawOutput) {\n    conversationHistory.appendModelOutput(rawAssistantOutput);\n  } else {\n    if (!rawAssistantOutput) {\n      debug(\n        'Raw assistant output is unavailable; saving only response text in conversation history.',\n      );\n    }\n    conversationHistory.appendMessage({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawAssistantOutput","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index","shouldReplayRawOutput"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;ACwBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,IAAAA,sBAAAA,MAAAA,AAAAA,EACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,AAAAA,IAAAA,qDAAAA,6BAAAA,AAAAA,EAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,IAAAA,mDAAAA,oBAAAA,AAAAA,EAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEU;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,IAAAA,sBAAAA,MAAAA,AAAAA,EACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,AAAAA,IAAAA,0BAAAA,2CAAAA,AAAAA,EAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,AAAAA,IAAAA,0BAAAA,2CAAAA,AAAAA,EAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,AAAAA,IAAAA,yBAAAA,iCAAAA,AAAAA,EAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,AAAAA,IAAAA,qCAAAA,iCAAAA,AAAAA,EAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAAqC;QACzC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMqD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,KAAKP;gBACP;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,KAAKP;YACP;SACD;IACH;IAEFX,oBAAoB,aAAa,CAACgB;IAGlChB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAA0B;QAC9B;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,EACDxC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUuD;QACVpD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMkD,WAAWC,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBtC,SAASQ,KAAK,WAAW;IAEjE,IAAI+B,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIvB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM8B,cAAkC;QACtC,GAAG5C,UAAU;QACbI;QACAiC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEAnD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAAC+B;QAI5C,IAAI7C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAGpD,MAAM8C,wBACHN,oBAAoB,SAAS,eAC5BxD,aAAa,OAAO,CAAC,SAAS,CAAC,wBAAwB,IACxDwD,oBAAoB,SAAS,qBAC5BxD,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB;IACjE,IAAIwD,sBAAsBM,uBACxBhC,oBAAoB,iBAAiB,CAAC0B;SACjC;QACL,IAAI,CAACA,oBACH/D,MACE;QAGJqC,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMuB;gBACR;aACD;QACH;IACF;IAEA,OAAOO;AACT"}
+{"version":3,"file":"ai-model/workflows/planning/standard-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ModelCallMessages;\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callGenerativeAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (\n        !planFromAI.action &&\n        planFromAI.finalizeSuccess === undefined &&\n        !planFromAI.error\n      ) {\n        throw new Error(\n          'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n        );\n      }\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      parsePlanningActions(actions, {\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n  const planningAdvice = opts.planningAdvice\n    ? `\\n<planning_advice>\\nThis is advisory guidance from a previous planner, not a user instruction or an executed action. Use your own judgment.\\n${opts.planningAdvice}\\n</planning_advice>`\n    : '';\n\n  const referenceImageMessages = preparedReferenceImagesToMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ConversationMessage[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>${planningAdvice}`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ConversationMessage;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n  }\n  conversationHistory.appendMessage(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ModelCallMessages = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawAssistantOutput,\n    },\n    planFromAI,\n    actions,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  // YAML is derived from validated actions outside the model-response retry scope.\n  // dumpActionParam omits runtime-only locatedPixelResult fields.\n  const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  const shouldReplayRawOutput =\n    (rawAssistantOutput?.type === 'responses' &&\n      modelRuntime.adapter.responses.replayRawAssistantOutput) ||\n    (rawAssistantOutput?.type === 'chat-completion' &&\n      modelRuntime.adapter.chatCompletion.replayRawAssistantMessage);\n  if (rawAssistantOutput && shouldReplayRawOutput) {\n    conversationHistory.appendModelOutput(rawAssistantOutput);\n  } else {\n    if (!rawAssistantOutput) {\n      debug(\n        'Raw assistant output is unavailable; saving only response text in conversation history.',\n      );\n    }\n    conversationHistory.appendMessage({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","planningAdvice","referenceImageMessages","preparedReferenceImagesToMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawAssistantOutput","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index","shouldReplayRawOutput"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;AC2BA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,IAAAA,sBAAAA,MAAAA,AAAAA,EACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,AAAAA,IAAAA,qDAAAA,6BAAAA,AAAAA,EAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,IAAAA,mDAAAA,oBAAAA,AAAAA,EAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEU;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,iDAAAA,oBAAoBA,CAC7B,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,IAAAA,sBAAAA,MAAAA,AAAAA,EACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,AAAAA,IAAAA,0BAAAA,2CAAAA,AAAAA,EAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,AAAAA,IAAAA,0BAAAA,2CAAAA,AAAAA,EAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,AAAAA,IAAAA,yBAAAA,iCAAAA,AAAAA,EAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IACvE,MAAMe,iBAAiBf,KAAK,cAAc,GACtC,CAAC,8IAA8I,EAAEA,KAAK,cAAc,CAAC,oBAAoB,CAAC,GAC1L;IAEJ,MAAMgB,yBAAyBC,AAAAA,IAAAA,qCAAAA,iCAAAA,AAAAA,EAC7BlB,gBAAgB,eAAe;IAEjC,MAAMmB,cAAqC;QACzC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGJ,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,EAAEgB,gBAAgB;gBACvG;aACD;QACH;WACGC;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBZ,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMmB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BlB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMsD,eAAepB,oBAAoB,cAAc;IACvD,MAAMqB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIpB,oBAAoB,sBAAsB,EAAE;QAC9CiB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGjB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEqB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,KAAKR;gBACP;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEiB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,KAAKR;YACP;SACD;IACH;IAEFX,oBAAoB,aAAa,CAACiB;IAGlCjB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMsB,aAAatB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMyB,OAA0B;QAC9B;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,EACDzC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUwD;QACVrD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMmD,WAAWC,AAAAA,IAAAA,mCAAAA,sBAAAA,AAAAA,EAAuBvC,SAASQ,KAAK,WAAW;IAEjE,IAAIgC,yBAAyB;IAG7B,IAAI5C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNmE,yBAAyB;QAEzB,IAAIxB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM+B,cAAkC;QACtC,GAAG7C,UAAU;QACbI;QACAkC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEApD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM8C,SAAS9C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAACgC;QAI5C,IAAI9C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAGpD,MAAM+C,wBACHN,oBAAoB,SAAS,eAC5BzD,aAAa,OAAO,CAAC,SAAS,CAAC,wBAAwB,IACxDyD,oBAAoB,SAAS,qBAC5BzD,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB;IACjE,IAAIyD,sBAAsBM,uBACxBjC,oBAAoB,iBAAiB,CAAC2B;SACjC;QACL,IAAI,CAACA,oBACHhE,MACE;QAGJqC,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMwB;gBACR;aACD;QACH;IACF;IAEA,OAAOO;AACT"}

package/dist/lib/ai-model/workflows/recorder-generation/playwright.js

@@ -141,7 +141,7 @@ ${constants_namespaceObject.PLAYWRIGHT_EXAMPLE_CODE}`;
             content: messageContent
         }
     ];
-    if (options.stream && options.onChunk) return await (0, external_service_caller_index_js_namespaceObject.callAI)(prompt, modelRuntime, {
+    if (options.stream && options.onChunk) return await (0, external_service_caller_index_js_namespaceObject.callGenerativeAI)(prompt, modelRuntime, {
         stream: true,
         onChunk: options.onChunk
     });

package/dist/lib/ai-model/workflows/recorder-generation/playwright.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/recorder-generation/playwright.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/recorder-generation/playwright.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getModelRuntime } from '../../models';\nimport { callAI, callAIWithStringResponse } from '../../service-caller';\n// Import shared utilities and types from yaml generation.\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './common';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAI(prompt, modelRuntime, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createModelRuntime","modelConfig","getModelRuntime","generatePlaywrightTest","events","options","modelRuntime","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","Error","generatePlaywrightTestStream","callAI"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;ACqDA,SAASI,mBAAmBC,WAAyB;IACnD,OAAOC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgBD;AACzB;AAKO,MAAME,yBAAyB,OACpCC,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,IAAAA,mCAAAA,cAAAA,AAAAA,EAAeH;IAGf,MAAMI,UAAUC,AAAAA,IAAAA,mCAAAA,mBAAAA,AAAAA,EAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;8LAWiJ,CAAC;IAG7L,MAAMK,iBAAiBC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,AAAAA,IAAAA,iDAAAA,wBAAAA,AAAAA,EAAyBF,QAAQb;IAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIE,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CnB,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,IAAAA,mCAAAA,cAAAA,AAAAA,EAAeH;IAGf,MAAMI,UAAUC,AAAAA,IAAAA,mCAAAA,mBAAAA,AAAAA,EAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;8LAYiJ,CAAC;IAG7L,MAAMK,iBAAiBC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMmB,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EAAOL,QAAQb,cAAc;QACxC,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,AAAAA,IAAAA,iDAAAA,wBAAAA,AAAAA,EAAyBF,QAAQb;QAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB;AACF"}
+{"version":3,"file":"ai-model/workflows/recorder-generation/playwright.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/recorder-generation/playwright.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getModelRuntime } from '../../models';\nimport {\n  callAIWithStringResponse,\n  callGenerativeAI,\n} from '../../service-caller';\n// Import shared utilities and types from yaml generation.\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './common';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callGenerativeAI(prompt, modelRuntime, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createModelRuntime","modelConfig","getModelRuntime","generatePlaywrightTest","events","options","modelRuntime","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","Error","generatePlaywrightTestStream","callGenerativeAI"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;ACwDA,SAASI,mBAAmBC,WAAyB;IACnD,OAAOC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgBD;AACzB;AAKO,MAAME,yBAAyB,OACpCC,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,IAAAA,mCAAAA,cAAAA,AAAAA,EAAeH;IAGf,MAAMI,UAAUC,AAAAA,IAAAA,mCAAAA,mBAAAA,AAAAA,EAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;8LAWiJ,CAAC;IAG7L,MAAMK,iBAAiBC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,AAAAA,IAAAA,iDAAAA,wBAAAA,AAAAA,EAAyBF,QAAQb;IAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIE,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CnB,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,IAAAA,mCAAAA,cAAAA,AAAAA,EAAeH;IAGf,MAAMI,UAAUC,AAAAA,IAAAA,mCAAAA,mBAAAA,AAAAA,EAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;8LAYiJ,CAAC;IAG7L,MAAMK,iBAAiBC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMmB,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EAAiBL,QAAQb,cAAc;QAClD,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,AAAAA,IAAAA,iDAAAA,wBAAAA,AAAAA,EAAyBF,QAAQb;QAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB;AACF"}

package/dist/lib/ai-model/workflows/recorder-generation/yaml.js

@@ -170,7 +170,7 @@ const generateRecorderYamlTestStream = async (input, options, modelConfig)=>{
     try {
         const prompt = createRecorderYamlPrompt(input);
         const modelRuntime = createModelRuntime(modelConfig);
-        if (options.stream && options.onChunk) return await (0, external_service_caller_index_js_namespaceObject.callAI)(prompt, modelRuntime, {
+        if (options.stream && options.onChunk) return await (0, external_service_caller_index_js_namespaceObject.callGenerativeAI)(prompt, modelRuntime, {
             stream: true,
             onChunk: options.onChunk
         });

package/dist/lib/ai-model/workflows/recorder-generation/yaml.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/recorder-generation/yaml.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/recorder-generation/yaml.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  type MidsceneRecorderTarget,\n  stringifyMidsceneRecorderTargetBlock,\n} from '@midscene/shared/recorder';\nimport { getModelRuntime } from '../../models';\nimport { callAI, callAIWithStringResponse } from '../../service-caller';\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type FilteredEvents,\n  type InputDescription,\n  type ProcessedEvent,\n  type RecorderGenerationContext,\n  type RecorderGenerationInput,\n  type RecorderGenerationOptions,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  prepareRecorderGenerationContext,\n  processEventsForLLM,\n  validateEvents,\n} from './common';\n\nexport type YamlGenerationOptions = RecorderGenerationOptions;\nexport type RecorderYamlGenerationInput = RecorderGenerationInput;\n\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  EventSummary,\n  FilteredEvents,\n  InputDescription,\n  ProcessedEvent,\n  RecorderGenerationContext,\n};\n\nexport {\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  prepareRecorderGenerationContext,\n  processEventsForLLM,\n  validateEvents,\n};\n\nconst getYamlLanguageInstruction = (language?: string) => {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable YAML content in ${normalizedLanguage}.\n- Keep YAML keys, field names, and Midscene API names unchanged.`;\n};\n\nconst createYamlPrompt = ({\n  yamlSummary,\n  screenshotAssets,\n  language,\n  targetBlock,\n  target,\n}: {\n  yamlSummary: EventSummary & { includeTimestamps: boolean };\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n  language?: string;\n  targetBlock: string;\n  target: MidsceneRecorderTarget;\n}): ConversationMessage[] => {\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n    },\n    {\n      role: 'user',\n      content: `Generate YAML test for Midscene.js automation from recorded events.\n\nTarget platform:\n- Preserve this exact top-level target platform: ${target.platformId}\n- Use exactly one top-level target block.\n- The target block must be:\n${targetBlock}\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nScreenshot assets:\n${JSON.stringify(\n  screenshotAssets.map((asset) => ({\n    eventIndex: asset.eventIndex,\n    eventHashId: asset.eventHashId,\n    eventType: asset.eventType,\n    relativePath: asset.relativePath,\n    description: yamlSummary.events[asset.eventIndex]?.description,\n  })),\n  null,\n  2,\n)}\n\nConvert events:\n- navigation → target URL or aiAction only when the target platform supports it\n- click → aiTap with the semantic element description\n- input → aiInput with value and semantic locate\n- scroll → aiScroll with appropriate direction and semantic scroll area\n- keydown → aiKeyboardPress\n- Add aiAssert for important state changes\n- Prefer event.semantic.replayInstruction and event.semantic.elementDescription when event.semantic.source is \"aiDescribe\" or \"recorderAI\" and event.semantic.status is \"ready\".\n- If event.semantic.source is \"heuristic\" or event.semantic.status is \"pending\"/\"failed\", use the screenshot/context to write the best visual instruction, and avoid raw coordinates unless there is no reliable semantic description.\n- Screenshot assets are context only. Use their eventIndex/eventHashId relationship to understand the matching event, but do not include screenshot file paths in the YAML unless the Midscene YAML API explicitly needs them.${getYamlLanguageInstruction(language)}\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n    },\n  ];\n\n  if (screenshotAssets.length > 0) {\n    prompt.push({\n      role: 'user',\n      content:\n        'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    prompt.push({\n      role: 'user',\n      content: screenshotAssets.flatMap((asset) => [\n        {\n          type: 'text',\n          text: `Screenshot asset for event #${asset.eventIndex + 1}: ${asset.relativePath}`,\n        },\n        {\n          type: 'image',\n          url: asset.dataUrl,\n        },\n      ]),\n    });\n  }\n\n  return prompt;\n};\n\nfunction createDefaultWebTarget(\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n): MidsceneRecorderTarget {\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const firstUrl =\n    options.navigationInfo?.urls?.find(Boolean) ||\n    navigationEvents.find((event) => event.url)?.url ||\n    '';\n  const firstViewport =\n    options.navigationInfo?.initialViewport ||\n    events.find((event) => event.pageInfo)?.pageInfo;\n\n  return {\n    platformId: 'web',\n    deviceId: firstUrl || undefined,\n    label: firstUrl || 'Web',\n    values: {\n      url: firstUrl,\n      ...(firstViewport?.width ? { viewportWidth: firstViewport.width } : {}),\n      ...(firstViewport?.height\n        ? { viewportHeight: firstViewport.height }\n        : {}),\n    },\n  };\n}\n\nfunction normalizeGeneratedYaml(content: string) {\n  const trimmed = content.trim();\n  const fencedMatch = trimmed.match(/^```(?:ya?ml)?\\s*([\\s\\S]*?)\\s*```$/i);\n  return `${(fencedMatch?.[1] ?? trimmed).trim()}\\n`;\n}\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\nfunction createRecorderYamlPrompt(\n  input: RecorderYamlGenerationInput,\n): ConversationMessage[] {\n  const { summary, screenshotAssets } = prepareRecorderGenerationContext(input);\n  const yamlSummary = {\n    ...summary,\n    target: input.target,\n    includeTimestamps: input.includeTimestamps || false,\n  };\n\n  return createYamlPrompt({\n    yamlSummary,\n    screenshotAssets,\n    language: input.language,\n    target: input.target,\n    targetBlock: stringifyMidsceneRecorderTargetBlock(input.target),\n  });\n}\n\nexport const generateRecorderYamlTest = async (\n  input: RecorderYamlGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  try {\n    const prompt = createRecorderYamlPrompt(input);\n    const response = await callAIWithStringResponse(\n      prompt,\n      createModelRuntime(modelConfig),\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return normalizeGeneratedYaml(response.content);\n    }\n\n    throw new Error('Failed to generate recorder YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder YAML test: ${error}`);\n  }\n};\n\nexport const generateRecorderYamlTestStream = async (\n  input: RecorderYamlGenerationInput,\n  options: StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  try {\n    const prompt = createRecorderYamlPrompt(input);\n    const modelRuntime = createModelRuntime(modelConfig);\n    if (options.stream && options.onChunk) {\n      return await callAI(prompt, modelRuntime, {\n        stream: true,\n        onChunk: options.onChunk,\n      });\n    }\n\n    const response = await callAIWithStringResponse(prompt, modelRuntime);\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: normalizeGeneratedYaml(response.content),\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate recorder YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder YAML test: ${error}`);\n  }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  return generateRecorderYamlTest(\n    {\n      ...options,\n      target: createDefaultWebTarget(events, options),\n      events,\n    },\n    modelConfig,\n  );\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  return generateRecorderYamlTestStream(\n    {\n      ...options,\n      target: createDefaultWebTarget(events, options),\n      events,\n    },\n    options,\n    modelConfig,\n  );\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getYamlLanguageInstruction","language","normalizedLanguage","createYamlPrompt","yamlSummary","screenshotAssets","targetBlock","target","prompt","YAML_EXAMPLE_CODE","JSON","asset","createDefaultWebTarget","events","options","navigationEvents","event","firstUrl","Boolean","firstViewport","undefined","normalizeGeneratedYaml","content","trimmed","fencedMatch","createModelRuntime","modelConfig","getModelRuntime","createRecorderYamlPrompt","input","summary","prepareRecorderGenerationContext","stringifyMidsceneRecorderTargetBlock","generateRecorderYamlTest","response","callAIWithStringResponse","Error","error","generateRecorderYamlTestStream","modelRuntime","callAI","generateYamlTest","generateYamlTestStream"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;ACsDA,MAAMI,6BAA6B,CAACC;IAClC,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;2CAEiC,EAAEA,mBAAmB;gEACA,CAAC;AACjE;AAEA,MAAMC,mBAAmB,CAAC,EACxBC,WAAW,EACXC,gBAAgB,EAChBJ,QAAQ,EACRK,WAAW,EACXC,MAAM,EAOP;IACC,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;QAC7I;QACA;YACE,MAAM;YACN,SAAS,CAAC;;;iDAGiC,EAAEF,OAAO,UAAU,CAAC;;;AAGrE,EAAED,YAAY;;;AAGd,EAAEI,KAAK,SAAS,CAACN,aAAa,MAAM,GAAG;;;AAGvC,EAAEM,KAAK,SAAS,CACdL,iBAAiB,GAAG,CAAC,CAACM,QAAW;oBAC/B,YAAYA,MAAM,UAAU;oBAC5B,aAAaA,MAAM,WAAW;oBAC9B,WAAWA,MAAM,SAAS;oBAC1B,cAAcA,MAAM,YAAY;oBAChC,aAAaP,YAAY,MAAM,CAACO,MAAM,UAAU,CAAC,EAAE;gBACrD,KACA,MACA,GACA;;;;;;;;;;;8NAW4N,EAAEX,2BAA2BC,UAAU;;8JAEvG,CAAC;QAC3J;KACD;IAED,IAAII,iBAAiB,MAAM,GAAG,GAAG;QAC/BG,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SACE;QACJ;QAEAA,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SAASH,iBAAiB,OAAO,CAAC,CAACM,QAAU;oBAC3C;wBACE,MAAM;wBACN,MAAM,CAAC,4BAA4B,EAAEA,MAAM,UAAU,GAAG,EAAE,EAAE,EAAEA,MAAM,YAAY,EAAE;oBACpF;oBACA;wBACE,MAAM;wBACN,KAAKA,MAAM,OAAO;oBACpB;iBACD;QACH;IACF;IAEA,OAAOH;AACT;AAEA,SAASI,uBACPC,MAA6B,EAC7BC,OAA8B;IAE9B,MAAMC,mBAAmBF,OAAO,MAAM,CACpC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,WACJH,QAAQ,cAAc,EAAE,MAAM,KAAKI,YACnCH,iBAAiB,IAAI,CAAC,CAACC,QAAUA,MAAM,GAAG,GAAG,OAC7C;IACF,MAAMG,gBACJL,QAAQ,cAAc,EAAE,mBACxBD,OAAO,IAAI,CAAC,CAACG,QAAUA,MAAM,QAAQ,GAAG;IAE1C,OAAO;QACL,YAAY;QACZ,UAAUC,YAAYG;QACtB,OAAOH,YAAY;QACnB,QAAQ;YACN,KAAKA;YACL,GAAIE,eAAe,QAAQ;gBAAE,eAAeA,cAAc,KAAK;YAAC,IAAI,CAAC,CAAC;YACtE,GAAIA,eAAe,SACf;gBAAE,gBAAgBA,cAAc,MAAM;YAAC,IACvC,CAAC,CAAC;QACR;IACF;AACF;AAEA,SAASE,uBAAuBC,OAAe;IAC7C,MAAMC,UAAUD,QAAQ,IAAI;IAC5B,MAAME,cAAcD,QAAQ,KAAK,CAAC;IAClC,OAAO,GAAIC,AAAAA,CAAAA,aAAa,CAAC,EAAE,IAAID,OAAM,EAAG,IAAI,GAAG,EAAE,CAAC;AACpD;AAEA,SAASE,mBAAmBC,WAAyB;IACnD,OAAOC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgBD;AACzB;AAEA,SAASE,yBACPC,KAAkC;IAElC,MAAM,EAAEC,OAAO,EAAEzB,gBAAgB,EAAE,GAAG0B,AAAAA,IAAAA,mCAAAA,gCAAAA,AAAAA,EAAiCF;IACvE,MAAMzB,cAAc;QAClB,GAAG0B,OAAO;QACV,QAAQD,MAAM,MAAM;QACpB,mBAAmBA,MAAM,iBAAiB,IAAI;IAChD;IAEA,OAAO1B,iBAAiB;QACtBC;QACAC;QACA,UAAUwB,MAAM,QAAQ;QACxB,QAAQA,MAAM,MAAM;QACpB,aAAaG,AAAAA,IAAAA,yBAAAA,oCAAAA,AAAAA,EAAqCH,MAAM,MAAM;IAChE;AACF;AAEO,MAAMI,2BAA2B,OACtCJ,OACAH;IAEA,IAAI;QACF,MAAMlB,SAASoB,yBAAyBC;QACxC,MAAMK,WAAW,MAAMC,AAAAA,IAAAA,iDAAAA,wBAAAA,AAAAA,EACrB3B,QACAiB,mBAAmBC;QAGrB,IAAIQ,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOb,uBAAuBa,SAAS,OAAO;QAGhD,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOC,OAAO;QACd,MAAM,IAAID,MAAM,CAAC,uCAAuC,EAAEC,OAAO;IACnE;AACF;AAEO,MAAMC,iCAAiC,OAC5CT,OACAf,SACAY;IAEA,IAAI;QACF,MAAMlB,SAASoB,yBAAyBC;QACxC,MAAMU,eAAed,mBAAmBC;QACxC,IAAIZ,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EACnC,OAAO,MAAM0B,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EAAOhC,QAAQ+B,cAAc;YACxC,QAAQ;YACR,SAASzB,QAAQ,OAAO;QAC1B;QAGF,MAAMoB,WAAW,MAAMC,AAAAA,IAAAA,iDAAAA,wBAAAA,AAAAA,EAAyB3B,QAAQ+B;QACxD,IAAIL,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASb,uBAAuBa,SAAS,OAAO;YAChD,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOC,OAAO;QACd,MAAM,IAAID,MAAM,CAAC,uCAAuC,EAAEC,OAAO;IACnE;AACF;AAOO,MAAMI,mBAAmB,OAC9B5B,QACAC,SACAY,cAEOO,yBACL;QACE,GAAGnB,OAAO;QACV,QAAQF,uBAAuBC,QAAQC;QACvCD;IACF,GACAa;AAOG,MAAMgB,yBAAyB,OACpC7B,QACAC,SACAY,cAEOY,+BACL;QACE,GAAGxB,OAAO;QACV,QAAQF,uBAAuBC,QAAQC;QACvCD;IACF,GACAC,SACAY"}
+{"version":3,"file":"ai-model/workflows/recorder-generation/yaml.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/recorder-generation/yaml.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  type MidsceneRecorderTarget,\n  stringifyMidsceneRecorderTargetBlock,\n} from '@midscene/shared/recorder';\nimport { getModelRuntime } from '../../models';\nimport {\n  callAIWithStringResponse,\n  callGenerativeAI,\n} from '../../service-caller';\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type FilteredEvents,\n  type InputDescription,\n  type ProcessedEvent,\n  type RecorderGenerationContext,\n  type RecorderGenerationInput,\n  type RecorderGenerationOptions,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  prepareRecorderGenerationContext,\n  processEventsForLLM,\n  validateEvents,\n} from './common';\n\nexport type YamlGenerationOptions = RecorderGenerationOptions;\nexport type RecorderYamlGenerationInput = RecorderGenerationInput;\n\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  EventSummary,\n  FilteredEvents,\n  InputDescription,\n  ProcessedEvent,\n  RecorderGenerationContext,\n};\n\nexport {\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  prepareRecorderGenerationContext,\n  processEventsForLLM,\n  validateEvents,\n};\n\nconst getYamlLanguageInstruction = (language?: string) => {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable YAML content in ${normalizedLanguage}.\n- Keep YAML keys, field names, and Midscene API names unchanged.`;\n};\n\nconst createYamlPrompt = ({\n  yamlSummary,\n  screenshotAssets,\n  language,\n  targetBlock,\n  target,\n}: {\n  yamlSummary: EventSummary & { includeTimestamps: boolean };\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n  language?: string;\n  targetBlock: string;\n  target: MidsceneRecorderTarget;\n}): ConversationMessage[] => {\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n    },\n    {\n      role: 'user',\n      content: `Generate YAML test for Midscene.js automation from recorded events.\n\nTarget platform:\n- Preserve this exact top-level target platform: ${target.platformId}\n- Use exactly one top-level target block.\n- The target block must be:\n${targetBlock}\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nScreenshot assets:\n${JSON.stringify(\n  screenshotAssets.map((asset) => ({\n    eventIndex: asset.eventIndex,\n    eventHashId: asset.eventHashId,\n    eventType: asset.eventType,\n    relativePath: asset.relativePath,\n    description: yamlSummary.events[asset.eventIndex]?.description,\n  })),\n  null,\n  2,\n)}\n\nConvert events:\n- navigation → target URL or aiAction only when the target platform supports it\n- click → aiTap with the semantic element description\n- input → aiInput with value and semantic locate\n- scroll → aiScroll with appropriate direction and semantic scroll area\n- keydown → aiKeyboardPress\n- Add aiAssert for important state changes\n- Prefer event.semantic.replayInstruction and event.semantic.elementDescription when event.semantic.source is \"aiDescribe\" or \"recorderAI\" and event.semantic.status is \"ready\".\n- If event.semantic.source is \"heuristic\" or event.semantic.status is \"pending\"/\"failed\", use the screenshot/context to write the best visual instruction, and avoid raw coordinates unless there is no reliable semantic description.\n- Screenshot assets are context only. Use their eventIndex/eventHashId relationship to understand the matching event, but do not include screenshot file paths in the YAML unless the Midscene YAML API explicitly needs them.${getYamlLanguageInstruction(language)}\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n    },\n  ];\n\n  if (screenshotAssets.length > 0) {\n    prompt.push({\n      role: 'user',\n      content:\n        'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    prompt.push({\n      role: 'user',\n      content: screenshotAssets.flatMap((asset) => [\n        {\n          type: 'text',\n          text: `Screenshot asset for event #${asset.eventIndex + 1}: ${asset.relativePath}`,\n        },\n        {\n          type: 'image',\n          url: asset.dataUrl,\n        },\n      ]),\n    });\n  }\n\n  return prompt;\n};\n\nfunction createDefaultWebTarget(\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n): MidsceneRecorderTarget {\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const firstUrl =\n    options.navigationInfo?.urls?.find(Boolean) ||\n    navigationEvents.find((event) => event.url)?.url ||\n    '';\n  const firstViewport =\n    options.navigationInfo?.initialViewport ||\n    events.find((event) => event.pageInfo)?.pageInfo;\n\n  return {\n    platformId: 'web',\n    deviceId: firstUrl || undefined,\n    label: firstUrl || 'Web',\n    values: {\n      url: firstUrl,\n      ...(firstViewport?.width ? { viewportWidth: firstViewport.width } : {}),\n      ...(firstViewport?.height\n        ? { viewportHeight: firstViewport.height }\n        : {}),\n    },\n  };\n}\n\nfunction normalizeGeneratedYaml(content: string) {\n  const trimmed = content.trim();\n  const fencedMatch = trimmed.match(/^```(?:ya?ml)?\\s*([\\s\\S]*?)\\s*```$/i);\n  return `${(fencedMatch?.[1] ?? trimmed).trim()}\\n`;\n}\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\nfunction createRecorderYamlPrompt(\n  input: RecorderYamlGenerationInput,\n): ConversationMessage[] {\n  const { summary, screenshotAssets } = prepareRecorderGenerationContext(input);\n  const yamlSummary = {\n    ...summary,\n    target: input.target,\n    includeTimestamps: input.includeTimestamps || false,\n  };\n\n  return createYamlPrompt({\n    yamlSummary,\n    screenshotAssets,\n    language: input.language,\n    target: input.target,\n    targetBlock: stringifyMidsceneRecorderTargetBlock(input.target),\n  });\n}\n\nexport const generateRecorderYamlTest = async (\n  input: RecorderYamlGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  try {\n    const prompt = createRecorderYamlPrompt(input);\n    const response = await callAIWithStringResponse(\n      prompt,\n      createModelRuntime(modelConfig),\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return normalizeGeneratedYaml(response.content);\n    }\n\n    throw new Error('Failed to generate recorder YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder YAML test: ${error}`);\n  }\n};\n\nexport const generateRecorderYamlTestStream = async (\n  input: RecorderYamlGenerationInput,\n  options: StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  try {\n    const prompt = createRecorderYamlPrompt(input);\n    const modelRuntime = createModelRuntime(modelConfig);\n    if (options.stream && options.onChunk) {\n      return await callGenerativeAI(prompt, modelRuntime, {\n        stream: true,\n        onChunk: options.onChunk,\n      });\n    }\n\n    const response = await callAIWithStringResponse(prompt, modelRuntime);\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: normalizeGeneratedYaml(response.content),\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate recorder YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder YAML test: ${error}`);\n  }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  return generateRecorderYamlTest(\n    {\n      ...options,\n      target: createDefaultWebTarget(events, options),\n      events,\n    },\n    modelConfig,\n  );\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  return generateRecorderYamlTestStream(\n    {\n      ...options,\n      target: createDefaultWebTarget(events, options),\n      events,\n    },\n    options,\n    modelConfig,\n  );\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getYamlLanguageInstruction","language","normalizedLanguage","createYamlPrompt","yamlSummary","screenshotAssets","targetBlock","target","prompt","YAML_EXAMPLE_CODE","JSON","asset","createDefaultWebTarget","events","options","navigationEvents","event","firstUrl","Boolean","firstViewport","undefined","normalizeGeneratedYaml","content","trimmed","fencedMatch","createModelRuntime","modelConfig","getModelRuntime","createRecorderYamlPrompt","input","summary","prepareRecorderGenerationContext","stringifyMidsceneRecorderTargetBlock","generateRecorderYamlTest","response","callAIWithStringResponse","Error","error","generateRecorderYamlTestStream","modelRuntime","callGenerativeAI","generateYamlTest","generateYamlTestStream"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;ACyDA,MAAMI,6BAA6B,CAACC;IAClC,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;2CAEiC,EAAEA,mBAAmB;gEACA,CAAC;AACjE;AAEA,MAAMC,mBAAmB,CAAC,EACxBC,WAAW,EACXC,gBAAgB,EAChBJ,QAAQ,EACRK,WAAW,EACXC,MAAM,EAOP;IACC,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;QAC7I;QACA;YACE,MAAM;YACN,SAAS,CAAC;;;iDAGiC,EAAEF,OAAO,UAAU,CAAC;;;AAGrE,EAAED,YAAY;;;AAGd,EAAEI,KAAK,SAAS,CAACN,aAAa,MAAM,GAAG;;;AAGvC,EAAEM,KAAK,SAAS,CACdL,iBAAiB,GAAG,CAAC,CAACM,QAAW;oBAC/B,YAAYA,MAAM,UAAU;oBAC5B,aAAaA,MAAM,WAAW;oBAC9B,WAAWA,MAAM,SAAS;oBAC1B,cAAcA,MAAM,YAAY;oBAChC,aAAaP,YAAY,MAAM,CAACO,MAAM,UAAU,CAAC,EAAE;gBACrD,KACA,MACA,GACA;;;;;;;;;;;8NAW4N,EAAEX,2BAA2BC,UAAU;;8JAEvG,CAAC;QAC3J;KACD;IAED,IAAII,iBAAiB,MAAM,GAAG,GAAG;QAC/BG,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SACE;QACJ;QAEAA,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SAASH,iBAAiB,OAAO,CAAC,CAACM,QAAU;oBAC3C;wBACE,MAAM;wBACN,MAAM,CAAC,4BAA4B,EAAEA,MAAM,UAAU,GAAG,EAAE,EAAE,EAAEA,MAAM,YAAY,EAAE;oBACpF;oBACA;wBACE,MAAM;wBACN,KAAKA,MAAM,OAAO;oBACpB;iBACD;QACH;IACF;IAEA,OAAOH;AACT;AAEA,SAASI,uBACPC,MAA6B,EAC7BC,OAA8B;IAE9B,MAAMC,mBAAmBF,OAAO,MAAM,CACpC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,WACJH,QAAQ,cAAc,EAAE,MAAM,KAAKI,YACnCH,iBAAiB,IAAI,CAAC,CAACC,QAAUA,MAAM,GAAG,GAAG,OAC7C;IACF,MAAMG,gBACJL,QAAQ,cAAc,EAAE,mBACxBD,OAAO,IAAI,CAAC,CAACG,QAAUA,MAAM,QAAQ,GAAG;IAE1C,OAAO;QACL,YAAY;QACZ,UAAUC,YAAYG;QACtB,OAAOH,YAAY;QACnB,QAAQ;YACN,KAAKA;YACL,GAAIE,eAAe,QAAQ;gBAAE,eAAeA,cAAc,KAAK;YAAC,IAAI,CAAC,CAAC;YACtE,GAAIA,eAAe,SACf;gBAAE,gBAAgBA,cAAc,MAAM;YAAC,IACvC,CAAC,CAAC;QACR;IACF;AACF;AAEA,SAASE,uBAAuBC,OAAe;IAC7C,MAAMC,UAAUD,QAAQ,IAAI;IAC5B,MAAME,cAAcD,QAAQ,KAAK,CAAC;IAClC,OAAO,GAAIC,AAAAA,CAAAA,aAAa,CAAC,EAAE,IAAID,OAAM,EAAG,IAAI,GAAG,EAAE,CAAC;AACpD;AAEA,SAASE,mBAAmBC,WAAyB;IACnD,OAAOC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgBD;AACzB;AAEA,SAASE,yBACPC,KAAkC;IAElC,MAAM,EAAEC,OAAO,EAAEzB,gBAAgB,EAAE,GAAG0B,AAAAA,IAAAA,mCAAAA,gCAAAA,AAAAA,EAAiCF;IACvE,MAAMzB,cAAc;QAClB,GAAG0B,OAAO;QACV,QAAQD,MAAM,MAAM;QACpB,mBAAmBA,MAAM,iBAAiB,IAAI;IAChD;IAEA,OAAO1B,iBAAiB;QACtBC;QACAC;QACA,UAAUwB,MAAM,QAAQ;QACxB,QAAQA,MAAM,MAAM;QACpB,aAAaG,AAAAA,IAAAA,yBAAAA,oCAAAA,AAAAA,EAAqCH,MAAM,MAAM;IAChE;AACF;AAEO,MAAMI,2BAA2B,OACtCJ,OACAH;IAEA,IAAI;QACF,MAAMlB,SAASoB,yBAAyBC;QACxC,MAAMK,WAAW,MAAMC,AAAAA,IAAAA,iDAAAA,wBAAAA,AAAAA,EACrB3B,QACAiB,mBAAmBC;QAGrB,IAAIQ,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOb,uBAAuBa,SAAS,OAAO;QAGhD,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOC,OAAO;QACd,MAAM,IAAID,MAAM,CAAC,uCAAuC,EAAEC,OAAO;IACnE;AACF;AAEO,MAAMC,iCAAiC,OAC5CT,OACAf,SACAY;IAEA,IAAI;QACF,MAAMlB,SAASoB,yBAAyBC;QACxC,MAAMU,eAAed,mBAAmBC;QACxC,IAAIZ,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EACnC,OAAO,MAAM0B,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EAAiBhC,QAAQ+B,cAAc;YAClD,QAAQ;YACR,SAASzB,QAAQ,OAAO;QAC1B;QAGF,MAAMoB,WAAW,MAAMC,AAAAA,IAAAA,iDAAAA,wBAAAA,AAAAA,EAAyB3B,QAAQ+B;QACxD,IAAIL,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASb,uBAAuBa,SAAS,OAAO;YAChD,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOC,OAAO;QACd,MAAM,IAAID,MAAM,CAAC,uCAAuC,EAAEC,OAAO;IACnE;AACF;AAOO,MAAMI,mBAAmB,OAC9B5B,QACAC,SACAY,cAEOO,yBACL;QACE,GAAGnB,OAAO;QACV,QAAQF,uBAAuBC,QAAQC;QACvCD;IACF,GACAa;AAOG,MAAMgB,yBAAyB,OACpC7B,QACAC,SACAY,cAEOY,+BACL;QACE,GAAGxB,OAAO;QACV,QAAQF,uBAAuBC,QAAQC;QACvCD;IACF,GACAC,SACAY"}

package/dist/lib/device/index.js

@@ -114,6 +114,7 @@ function pointFromLocate(locate, missingMessage) {
 function defineLocatedPointAction(config) {
     return defineAction({
         name: config.name,
+        supportedBySystemOne: config.supportedBySystemOne,
         description: config.description,
         interfaceAlias: config.interfaceAlias,
         paramSchema: config.paramSchema,
@@ -128,6 +129,7 @@ const actionTapParamSchema = external_zod_namespaceObject.z.object({
 });
 const defineActionTap = (tap)=>defineLocatedPointAction({
         name: 'Tap',
+        supportedBySystemOne: true,
         description: 'Tap the element',
         interfaceAlias: 'aiTap',
         paramSchema: actionTapParamSchema,
@@ -166,6 +168,7 @@ const actionRightClickParamSchema = external_zod_namespaceObject.z.object({
 });
 const defineActionRightClick = (rightClick)=>defineLocatedPointAction({
         name: 'RightClick',
+        supportedBySystemOne: true,
         description: 'Right click the element',
         interfaceAlias: 'aiRightClick',
         paramSchema: actionRightClickParamSchema,
@@ -184,6 +187,7 @@ const actionDoubleClickParamSchema = external_zod_namespaceObject.z.object({
 });
 const defineActionDoubleClick = (doubleClick)=>defineLocatedPointAction({
         name: 'DoubleClick',
+        supportedBySystemOne: true,
         description: 'Double click the element',
         interfaceAlias: 'aiDoubleClick',
         paramSchema: actionDoubleClickParamSchema,
@@ -202,6 +206,7 @@ const actionHoverParamSchema = external_zod_namespaceObject.z.object({
 });
 const defineActionHover = (hover)=>defineLocatedPointAction({
         name: 'Hover',
+        supportedBySystemOne: true,
         description: 'Move the mouse to the element',
         interfaceAlias: 'aiHover',
         paramSchema: actionHoverParamSchema,
@@ -448,6 +453,7 @@ const actionClearInputParamSchema = external_zod_namespaceObject.z.object({
 });
 const defineActionClearInput = (clearInput)=>defineAction({
         name: 'ClearInput',
+        supportedBySystemOne: true,
         description: inputLocateDescription,
         interfaceAlias: 'aiClearInput',
         paramSchema: actionClearInputParamSchema,
@@ -543,6 +549,7 @@ function normalizePinchParam(param, screenSize) {
 function defineSystemInputAction(config, call) {
     return defineAction({
         name: config.name,
+        supportedBySystemOne: config.supportedBySystemOne,
         description: config.description,
         interfaceAlias: config.interfaceAlias,
         delayBeforeRunner: config.delayBeforeRunner,