@midscene/core 1.9.6 → 1.9.7

package/dist/lib/ai-model/prompt/markdown-generator.js

@@ -28,10 +28,91 @@ __webpack_require__.d(__webpack_exports__, {
     generateRecorderMarkdownReplay: ()=>generateRecorderMarkdownReplay,
     convertRecordLogIntoMarkdown: ()=>convertRecordLogIntoMarkdown
 });
+const img_namespaceObject = require("@midscene/shared/img");
+const logger_namespaceObject = require("@midscene/shared/logger");
 const recorder_namespaceObject = require("@midscene/shared/recorder");
 const external_index_js_namespaceObject = require("../index.js");
 const index_js_namespaceObject = require("../models/index.js");
 const external_recorder_generation_common_js_namespaceObject = require("./recorder-generation-common.js");
+const MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET = 600000;
+const MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE = 768;
+const debugMarkdownReplay = (0, logger_namespaceObject.getDebug)('ai:recorder-markdown', {
+    console: true
+});
+function limitScreenshotAssetsForMarkdownReplay(screenshotAssets) {
+    let usedPayload = 0;
+    return screenshotAssets.filter((asset)=>{
+        const payloadSize = asset.dataUrl.length;
+        if (payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET || usedPayload + payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET) return false;
+        usedPayload += payloadSize;
+        return true;
+    });
+}
+async function compressScreenshotAssetForMarkdownReplay(asset) {
+    const { width, height } = await (0, img_namespaceObject.imageInfoOfBase64)(asset.dataUrl);
+    const longestEdge = Math.max(width, height);
+    if (longestEdge <= MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE) return asset;
+    const scale = MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE / longestEdge;
+    const dataUrl = await (0, img_namespaceObject.resizeImgBase64)(asset.dataUrl, {
+        width: Math.max(1, Math.round(width * scale)),
+        height: Math.max(1, Math.round(height * scale))
+    });
+    const { body, mimeType } = (0, img_namespaceObject.parseBase64)(dataUrl);
+    return {
+        ...asset,
+        dataUrl,
+        base64Data: body,
+        mimeType
+    };
+}
+async function prepareScreenshotAssetsForMarkdownReplay(screenshotAssets) {
+    const compressedAssets = [];
+    for (const asset of screenshotAssets)try {
+        compressedAssets.push(await compressScreenshotAssetForMarkdownReplay(asset));
+    } catch  {
+        compressedAssets.push(asset);
+    }
+    return limitScreenshotAssetsForMarkdownReplay(compressedAssets);
+}
+function summarizeScreenshotAssets(screenshotAssets) {
+    const payloadSizes = screenshotAssets.map((asset)=>asset.dataUrl.length);
+    return {
+        count: screenshotAssets.length,
+        totalPayloadChars: payloadSizes.reduce((sum, size)=>sum + size, 0),
+        maxPayloadChars: payloadSizes.length ? Math.max(...payloadSizes) : 0
+    };
+}
+function getPromptShape(prompt) {
+    let textChars = 0;
+    let imageCount = 0;
+    for (const message of prompt){
+        const content = message.content;
+        if ('string' == typeof content) {
+            textChars += content.length;
+            continue;
+        }
+        if (Array.isArray(content)) {
+            for (const part of content)if ('object' == typeof part && part && 'type' in part) {
+                if ('text' === part.type && 'text' in part) textChars += String(part.text).length;
+                if ('image_url' === part.type) imageCount += 1;
+            }
+        }
+    }
+    return {
+        textChars,
+        imageCount
+    };
+}
+function removeOmittedScreenshotPaths(summary, screenshotAssets) {
+    const includedScreenshotPaths = new Set(screenshotAssets.map((asset)=>asset.relativePath));
+    return {
+        ...summary,
+        events: summary.events.map((event)=>event.screenshotPath && !includedScreenshotPaths.has(event.screenshotPath) ? {
+                ...event,
+                screenshotPath: void 0
+            } : event)
+    };
+}
 function getMarkdownLanguageInstruction(language) {
     const normalizedLanguage = language?.trim();
     if (!normalizedLanguage) return '';
@@ -51,17 +132,62 @@ function resolveModelRuntime(model) {
 }
 function createRecorderMarkdownReplayPrompt(input) {
     (0, external_recorder_generation_common_js_namespaceObject.validateEvents)(input.events);
-    const { summary, screenshotAssets } = (0, external_recorder_generation_common_js_namespaceObject.prepareRecorderGenerationContext)(input);
-    const promptText = `Generate a Markdown replay script for Midscene Agent.
-
-This Markdown will be executed with:
-await agent.runMarkdown('./x.md')
+    const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } = (0, external_recorder_generation_common_js_namespaceObject.prepareRecorderGenerationContext)(input);
+    const screenshotAssets = limitScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);
+    const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);
+    return createRecorderMarkdownReplayPromptFromContext(input, summary, screenshotAssets);
+}
+async function createRecorderMarkdownReplayPromptForGeneration(input) {
+    (0, external_recorder_generation_common_js_namespaceObject.validateEvents)(input.events);
+    const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } = (0, external_recorder_generation_common_js_namespaceObject.prepareRecorderGenerationContext)(input);
+    const screenshotAssets = await prepareScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);
+    const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);
+    const prompt = createRecorderMarkdownReplayPromptFromContext(input, summary, screenshotAssets);
+    debugMarkdownReplay('markdown replay prompt shape %o', {
+        eventCount: input.events.length,
+        maxScreenshots: input.maxScreenshots,
+        rawScreenshots: summarizeScreenshotAssets(rawScreenshotAssets),
+        includedScreenshots: summarizeScreenshotAssets(screenshotAssets),
+        prompt: getPromptShape(prompt)
+    });
+    return prompt;
+}
+function createRecorderMarkdownReplayPromptFromContext(input, summary, screenshotAssets) {
+    const screenshotIndexByEventHash = new Map(screenshotAssets.map((asset, index)=>[
+            asset.eventHashId,
+            `screenshot-${index + 1}`
+        ]));
+    const events = summary.events.map((event)=>{
+        const screenshotRef = screenshotIndexByEventHash.get(event.hashId);
+        const { screenshotPath, ...eventWithoutScreenshotPath } = event;
+        return screenshotRef ? {
+            ...eventWithoutScreenshotPath,
+            screenshotRef
+        } : eventWithoutScreenshotPath;
+    });
+    const promptPayload = {
+        testName: input.testName || summary.testName,
+        target: {
+            platformId: input.target.platformId,
+            label: input.target.label,
+            values: input.target.values
+        },
+        startUrl: summary.startUrl,
+        events,
+        screenshots: screenshotAssets.map((asset, index)=>({
+                screenshotRef: `screenshot-${index + 1}`,
+                eventIndex: asset.eventIndex,
+                eventHashId: asset.eventHashId,
+                eventType: asset.eventType,
+                description: (0, recorder_namespaceObject.getMidsceneRecorderEventDescription)(input.events[asset.eventIndex])
+            }))
+    };
+    const promptText = `Generate a Markdown replay script for Midscene Agent. It will be executed with:
+await agent.aiAct(markdownReplayPrompt)
 
-It is an AI-executable replay script, not a human report.
+Use only the recorder data and screenshots below.
 
-Target platform:
-- Preserve this exact platform: ${input.target.platformId}
-- Target block:
+Target block:
 ${(0, recorder_namespaceObject.stringifyMidsceneRecorderTargetBlock)(input.target)}
 
 Replay goal:
@@ -71,13 +197,18 @@ Replay goal:
 - Do not invent alternative navigation paths.
 - Do not skip, merge, reorder, or add extra user actions.
 - Prefer recorded UI text, element descriptions, URLs, input values, and scroll direction.
-- Prefer event.replayInstruction and event.elementDescription when descriptionSource is "ai".
-- If descriptionSource is "fallback", use the screenshot/context to write the best visual instruction.
+- For input events, enter event.typedText/event.value exactly; do not infer or correct the text from screenshots.
+- Prefer event.semantic.replayInstruction and event.semantic.elementDescription when event.semantic.source is "aiDescribe" or "recorderAI" and event.semantic.status is "ready".
+- For scroll events, preserve the recorded scroll region from event.semantic.elementDescription/replayInstruction. If the scroll happened in a specific panel, list, table, dialog body, menu, navigation area, or content pane, keep that region in the Markdown step instead of generalizing it to the whole page.
+- If event.semantic.source is "heuristic" or event.semantic.status is "pending"/"failed", use the screenshot/context to write the best visual instruction.
 - Coordinates are only fallback hints. Do not make coordinates the primary instruction when text or screenshots are available.
+- For a click/tap that only focuses a field before an input event, describe the target as the field/control itself. Do not target a placeholder character, typed character, caret, or inner text fragment inside the field.
 - If a target cannot be found, stop and report the missing step. Do not click similar-looking elements.
-- Use screenshots only when they are provided below. Reference them by their exact relative paths.
+- Screenshots are only generation-time visual evidence for you. The generated Markdown will be passed directly to agent.aiAct(markdownReplayPrompt), which accepts text only and cannot receive attached images.
+- Convert any useful screenshot evidence into textual replay instructions. Do not include screenshots, image syntax, image paths, or reference-image names in the generated Markdown.
+- Never write Markdown image syntax such as ![step context](...), reference-style images, HTML <img> tags, ./screenshots/... paths, or screenshot-* names in the output.
 
-Required Markdown structure:
+Required structure:
 # ${input.testName || summary.testName}
 
 ## Goal
@@ -87,33 +218,11 @@ Reproduce the recorded user workflow exactly.
 - Platform: ${input.target.platformId}
 - Start target: ${summary.startUrl || input.target.label || input.target.deviceId || 'Recorded target'}
 
-## Replay rules
-- Follow the steps in order.
-- Do not invent alternative navigation paths.
-- If a referenced target cannot be found, stop and report the missing step.
-
 ## Steps
 1. ...
 
-Event summary:
-${JSON.stringify({
-        ...summary,
-        target: input.target,
-        events: summary.events,
-        screenshotAssets: screenshotAssets.map((asset)=>({
-                eventIndex: asset.eventIndex,
-                eventHashId: asset.eventHashId,
-                eventType: asset.eventType,
-                relativePath: asset.relativePath,
-                description: (0, recorder_namespaceObject.getMidsceneRecorderEventDescription)(input.events[asset.eventIndex])
-            }))
-    }, null, 2)}
-
-Screenshot rules:
-- Insert a screenshot directly under the step that needs visual grounding.
-- Use Markdown image syntax exactly like: ![step context](./screenshots/event-001-click.png)
-- Only reference paths listed in screenshotAssets.
-- Do not reference images that are not listed.${getMarkdownLanguageInstruction(input.language)}
+Recorder data:
+${JSON.stringify(promptPayload, null, 2)}${getMarkdownLanguageInstruction(input.language)}
 
 Important: Return ONLY raw Markdown. Do NOT wrap the response in markdown code blocks.`;
     const content = [
@@ -123,9 +232,10 @@ Important: Return ONLY raw Markdown. Do NOT wrap the response in markdown code b
         }
     ];
     for (const asset of screenshotAssets){
+        const screenshotRef = screenshotIndexByEventHash.get(asset.eventHashId);
         content.push({
             type: 'text',
-            text: `Screenshot asset for event #${asset.eventIndex + 1}: ${asset.relativePath}`
+            text: `${screenshotRef} for event #${asset.eventIndex + 1}`
         });
         content.push({
             type: 'image_url',
@@ -137,7 +247,7 @@ Important: Return ONLY raw Markdown. Do NOT wrap the response in markdown code b
     return [
         {
             role: 'system',
-            content: "You generate precise Markdown replay scripts for Midscene agent.runMarkdown. The output must be deterministic, ordered, and safe for AI execution."
+            content: "You generate precise Markdown replay scripts for Midscene agent.aiAct. The final output is plain text that will be passed directly to agent.aiAct, so it must be deterministic, ordered, safe for AI execution, and must not contain image references, screenshot paths, or screenshot labels."
         },
         {
             role: 'user',
@@ -147,7 +257,7 @@ Important: Return ONLY raw Markdown. Do NOT wrap the response in markdown code b
 }
 async function generateRecorderMarkdownReplay(input, model) {
     try {
-        const prompt = createRecorderMarkdownReplayPrompt(input);
+        const prompt = await createRecorderMarkdownReplayPromptForGeneration(input);
         const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, resolveModelRuntime(model));
         if (response?.content && 'string' == typeof response.content) return normalizeGeneratedMarkdown(response.content);
         throw new Error('Failed to generate recorder Markdown replay');

package/dist/lib/ai-model/prompt/markdown-generator.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/markdown-generator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/markdown-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { IModelConfig } from '@midscene/shared/env';\nimport {\n  getMidsceneRecorderEventDescription,\n  stringifyMidsceneRecorderTargetBlock,\n} from '@midscene/shared/recorder';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { callAIWithStringResponse } from '../index';\nimport { type ModelRuntime, getModelRuntime } from '../models';\nimport {\n  type RecorderGenerationInput,\n  prepareRecorderGenerationContext,\n  validateEvents,\n} from './recorder-generation-common';\n\nexport type RecorderMarkdownGenerationInput = RecorderGenerationInput;\n\nfunction getMarkdownLanguageInstruction(language?: string) {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable Markdown instructions in ${normalizedLanguage}.\n- Keep file paths, URLs, platform ids, API names, and quoted UI text unchanged.`;\n}\n\nfunction normalizeGeneratedMarkdown(content: string) {\n  const trimmed = content.trim();\n  const fencedMatch = trimmed.match(\n    /^```(?:md|markdown)?\\s*([\\s\\S]*?)\\s*```$/i,\n  );\n  return `${(fencedMatch?.[1] ?? trimmed).trim()}\\n`;\n}\n\nfunction resolveModelRuntime(model: IModelConfig | ModelRuntime): ModelRuntime {\n  if ('config' in model && 'adapter' in model) {\n    return model;\n  }\n  return getModelRuntime(model);\n}\n\nexport function createRecorderMarkdownReplayPrompt(\n  input: RecorderMarkdownGenerationInput,\n): ChatCompletionMessageParam[] {\n  validateEvents(input.events);\n\n  const { summary, screenshotAssets } = prepareRecorderGenerationContext(input);\n  const promptText = `Generate a Markdown replay script for Midscene Agent.\n\nThis Markdown will be executed with:\nawait agent.runMarkdown('./x.md')\n\nIt is an AI-executable replay script, not a human report.\n\nTarget platform:\n- Preserve this exact platform: ${input.target.platformId}\n- Target block:\n${stringifyMidsceneRecorderTargetBlock(input.target)}\n\nReplay goal:\n- Reproduce the recorded user workflow exactly.\n- Preserve event order.\n- Preserve the user's original intent.\n- Do not invent alternative navigation paths.\n- Do not skip, merge, reorder, or add extra user actions.\n- Prefer recorded UI text, element descriptions, URLs, input values, and scroll direction.\n- Prefer event.replayInstruction and event.elementDescription when descriptionSource is \"ai\".\n- If descriptionSource is \"fallback\", use the screenshot/context to write the best visual instruction.\n- Coordinates are only fallback hints. Do not make coordinates the primary instruction when text or screenshots are available.\n- If a target cannot be found, stop and report the missing step. Do not click similar-looking elements.\n- Use screenshots only when they are provided below. Reference them by their exact relative paths.\n\nRequired Markdown structure:\n# ${input.testName || summary.testName}\n\n## Goal\nReproduce the recorded user workflow exactly.\n\n## Target\n- Platform: ${input.target.platformId}\n- Start target: ${summary.startUrl || input.target.label || input.target.deviceId || 'Recorded target'}\n\n## Replay rules\n- Follow the steps in order.\n- Do not invent alternative navigation paths.\n- If a referenced target cannot be found, stop and report the missing step.\n\n## Steps\n1. ...\n\nEvent summary:\n${JSON.stringify(\n  {\n    ...summary,\n    target: input.target,\n    events: summary.events,\n    screenshotAssets: screenshotAssets.map((asset) => ({\n      eventIndex: asset.eventIndex,\n      eventHashId: asset.eventHashId,\n      eventType: asset.eventType,\n      relativePath: asset.relativePath,\n      description: getMidsceneRecorderEventDescription(\n        input.events[asset.eventIndex],\n      ),\n    })),\n  },\n  null,\n  2,\n)}\n\nScreenshot rules:\n- Insert a screenshot directly under the step that needs visual grounding.\n- Use Markdown image syntax exactly like: ![step context](./screenshots/event-001-click.png)\n- Only reference paths listed in screenshotAssets.\n- Do not reference images that are not listed.${getMarkdownLanguageInstruction(input.language)}\n\nImportant: Return ONLY raw Markdown. Do NOT wrap the response in markdown code blocks.`;\n\n  const content: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  for (const asset of screenshotAssets) {\n    content.push({\n      type: 'text',\n      text: `Screenshot asset for event #${asset.eventIndex + 1}: ${asset.relativePath}`,\n    });\n    content.push({\n      type: 'image_url',\n      image_url: {\n        url: asset.dataUrl,\n      },\n    });\n  }\n\n  return [\n    {\n      role: 'system',\n      content:\n        'You generate precise Markdown replay scripts for Midscene agent.runMarkdown. The output must be deterministic, ordered, and safe for AI execution.',\n    },\n    {\n      role: 'user',\n      content,\n    },\n  ];\n}\n\nexport async function generateRecorderMarkdownReplay(\n  input: RecorderMarkdownGenerationInput,\n  model: IModelConfig | ModelRuntime,\n): Promise<string> {\n  try {\n    const prompt = createRecorderMarkdownReplayPrompt(input);\n    const response = await callAIWithStringResponse(\n      prompt,\n      resolveModelRuntime(model),\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return normalizeGeneratedMarkdown(response.content);\n    }\n\n    throw new Error('Failed to generate recorder Markdown replay');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder Markdown replay: ${error}`);\n  }\n}\n\nexport async function convertRecordLogIntoMarkdown(\n  log: RecorderMarkdownGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> {\n  return generateRecorderMarkdownReplay(log, modelConfig);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getMarkdownLanguageInstruction","language","normalizedLanguage","normalizeGeneratedMarkdown","content","trimmed","fencedMatch","resolveModelRuntime","model","getModelRuntime","createRecorderMarkdownReplayPrompt","input","validateEvents","summary","screenshotAssets","prepareRecorderGenerationContext","promptText","stringifyMidsceneRecorderTargetBlock","JSON","asset","getMidsceneRecorderEventDescription","generateRecorderMarkdownReplay","prompt","response","callAIWithStringResponse","Error","error","convertRecordLogIntoMarkdown","log","modelConfig"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACUA,SAASI,+BAA+BC,QAAiB;IACvD,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;oDAE0C,EAAEA,mBAAmB;+EACM,CAAC;AAChF;AAEA,SAASC,2BAA2BC,OAAe;IACjD,MAAMC,UAAUD,QAAQ,IAAI;IAC5B,MAAME,cAAcD,QAAQ,KAAK,CAC/B;IAEF,OAAO,GAAIC,AAAAA,CAAAA,aAAa,CAAC,EAAE,IAAID,OAAM,EAAG,IAAI,GAAG,EAAE,CAAC;AACpD;AAEA,SAASE,oBAAoBC,KAAkC;IAC7D,IAAI,YAAYA,SAAS,aAAaA,OACpC,OAAOA;IAET,OAAOC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgBD;AACzB;AAEO,SAASE,mCACdC,KAAsC;IAEtCC,IAAAA,uDAAAA,cAAAA,AAAAA,EAAeD,MAAM,MAAM;IAE3B,MAAM,EAAEE,OAAO,EAAEC,gBAAgB,EAAE,GAAGC,AAAAA,IAAAA,uDAAAA,gCAAAA,AAAAA,EAAiCJ;IACvE,MAAMK,aAAa,CAAC;;;;;;;;gCAQU,EAAEL,MAAM,MAAM,CAAC,UAAU,CAAC;;AAE1D,EAAEM,AAAAA,IAAAA,yBAAAA,oCAAAA,AAAAA,EAAqCN,MAAM,MAAM,EAAE;;;;;;;;;;;;;;;;EAgBnD,EAAEA,MAAM,QAAQ,IAAIE,QAAQ,QAAQ,CAAC;;;;;;YAM3B,EAAEF,MAAM,MAAM,CAAC,UAAU,CAAC;gBACtB,EAAEE,QAAQ,QAAQ,IAAIF,MAAM,MAAM,CAAC,KAAK,IAAIA,MAAM,MAAM,CAAC,QAAQ,IAAI,kBAAkB;;;;;;;;;;;AAWvG,EAAEO,KAAK,SAAS,CACd;QACE,GAAGL,OAAO;QACV,QAAQF,MAAM,MAAM;QACpB,QAAQE,QAAQ,MAAM;QACtB,kBAAkBC,iBAAiB,GAAG,CAAC,CAACK,QAAW;gBACjD,YAAYA,MAAM,UAAU;gBAC5B,aAAaA,MAAM,WAAW;gBAC9B,WAAWA,MAAM,SAAS;gBAC1B,cAAcA,MAAM,YAAY;gBAChC,aAAaC,AAAAA,IAAAA,yBAAAA,mCAAAA,AAAAA,EACXT,MAAM,MAAM,CAACQ,MAAM,UAAU,CAAC;YAElC;IACF,GACA,MACA,GACA;;;;;;8CAM4C,EAAEnB,+BAA+BW,MAAM,QAAQ,EAAE;;sFAET,CAAC;IAErF,MAAMP,UAAiB;QACrB;YACE,MAAM;YACN,MAAMY;QACR;KACD;IAED,KAAK,MAAMG,SAASL,iBAAkB;QACpCV,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,CAAC,4BAA4B,EAAEe,MAAM,UAAU,GAAG,EAAE,EAAE,EAAEA,MAAM,YAAY,EAAE;QACpF;QACAf,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,WAAW;gBACT,KAAKe,MAAM,OAAO;YACpB;QACF;IACF;IAEA,OAAO;QACL;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACNf;QACF;KACD;AACH;AAEO,eAAeiB,+BACpBV,KAAsC,EACtCH,KAAkC;IAElC,IAAI;QACF,MAAMc,SAASZ,mCAAmCC;QAClD,MAAMY,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACrBF,QACAf,oBAAoBC;QAGtB,IAAIe,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOpB,2BAA2BoB,SAAS,OAAO;QAGpD,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOC,OAAO;QACd,MAAM,IAAID,MAAM,CAAC,6CAA6C,EAAEC,OAAO;IACzE;AACF;AAEO,eAAeC,6BACpBC,GAAoC,EACpCC,WAAyB;IAEzB,OAAOR,+BAA+BO,KAAKC;AAC7C"}
+{"version":3,"file":"ai-model/prompt/markdown-generator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/markdown-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { IModelConfig } from '@midscene/shared/env';\nimport {\n  imageInfoOfBase64,\n  parseBase64,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport {\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  getMidsceneRecorderEventDescription,\n  stringifyMidsceneRecorderTargetBlock,\n} from '@midscene/shared/recorder';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { callAIWithStringResponse } from '../index';\nimport { type ModelRuntime, getModelRuntime } from '../models';\nimport {\n  type RecorderGenerationInput,\n  prepareRecorderGenerationContext,\n  validateEvents,\n} from './recorder-generation-common';\n\nexport type RecorderMarkdownGenerationInput = RecorderGenerationInput;\n\nconst MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET = 600_000;\nconst MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE = 768;\nconst debugMarkdownReplay = getDebug('ai:recorder-markdown', {\n  console: true,\n});\n\nfunction limitScreenshotAssetsForMarkdownReplay(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  let usedPayload = 0;\n  return screenshotAssets.filter((asset) => {\n    const payloadSize = asset.dataUrl.length;\n    if (\n      payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET ||\n      usedPayload + payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET\n    ) {\n      return false;\n    }\n    usedPayload += payloadSize;\n    return true;\n  });\n}\n\nasync function compressScreenshotAssetForMarkdownReplay(\n  asset: MidsceneRecorderMarkdownScreenshotAsset,\n): Promise<MidsceneRecorderMarkdownScreenshotAsset> {\n  const { width, height } = await imageInfoOfBase64(asset.dataUrl);\n  const longestEdge = Math.max(width, height);\n  if (longestEdge <= MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE) {\n    return asset;\n  }\n\n  const scale = MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE / longestEdge;\n  const dataUrl = await resizeImgBase64(asset.dataUrl, {\n    width: Math.max(1, Math.round(width * scale)),\n    height: Math.max(1, Math.round(height * scale)),\n  });\n  const { body, mimeType } = parseBase64(dataUrl);\n  return {\n    ...asset,\n    dataUrl,\n    base64Data: body,\n    mimeType,\n  };\n}\n\nasync function prepareScreenshotAssetsForMarkdownReplay(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const compressedAssets: MidsceneRecorderMarkdownScreenshotAsset[] = [];\n  for (const asset of screenshotAssets) {\n    try {\n      compressedAssets.push(\n        await compressScreenshotAssetForMarkdownReplay(asset),\n      );\n    } catch {\n      compressedAssets.push(asset);\n    }\n  }\n  return limitScreenshotAssetsForMarkdownReplay(compressedAssets);\n}\n\nfunction summarizeScreenshotAssets(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const payloadSizes = screenshotAssets.map((asset) => asset.dataUrl.length);\n  return {\n    count: screenshotAssets.length,\n    totalPayloadChars: payloadSizes.reduce((sum, size) => sum + size, 0),\n    maxPayloadChars: payloadSizes.length ? Math.max(...payloadSizes) : 0,\n  };\n}\n\nfunction getPromptShape(prompt: ChatCompletionMessageParam[]) {\n  let textChars = 0;\n  let imageCount = 0;\n  for (const message of prompt) {\n    const content = message.content;\n    if (typeof content === 'string') {\n      textChars += content.length;\n      continue;\n    }\n    if (!Array.isArray(content)) {\n      continue;\n    }\n    for (const part of content) {\n      if (typeof part === 'object' && part && 'type' in part) {\n        if (part.type === 'text' && 'text' in part) {\n          textChars += String(part.text).length;\n        }\n        if (part.type === 'image_url') {\n          imageCount += 1;\n        }\n      }\n    }\n  }\n  return { textChars, imageCount };\n}\n\nfunction removeOmittedScreenshotPaths(\n  summary: ReturnType<typeof prepareRecorderGenerationContext>['summary'],\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const includedScreenshotPaths = new Set(\n    screenshotAssets.map((asset) => asset.relativePath),\n  );\n  return {\n    ...summary,\n    events: summary.events.map((event) =>\n      event.screenshotPath && !includedScreenshotPaths.has(event.screenshotPath)\n        ? { ...event, screenshotPath: undefined }\n        : event,\n    ),\n  };\n}\n\nfunction getMarkdownLanguageInstruction(language?: string) {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable Markdown instructions in ${normalizedLanguage}.\n- Keep file paths, URLs, platform ids, API names, and quoted UI text unchanged.`;\n}\n\nfunction normalizeGeneratedMarkdown(content: string) {\n  const trimmed = content.trim();\n  const fencedMatch = trimmed.match(\n    /^```(?:md|markdown)?\\s*([\\s\\S]*?)\\s*```$/i,\n  );\n  return `${(fencedMatch?.[1] ?? trimmed).trim()}\\n`;\n}\n\nfunction resolveModelRuntime(model: IModelConfig | ModelRuntime): ModelRuntime {\n  if ('config' in model && 'adapter' in model) {\n    return model;\n  }\n  return getModelRuntime(model);\n}\n\nexport function createRecorderMarkdownReplayPrompt(\n  input: RecorderMarkdownGenerationInput,\n): ChatCompletionMessageParam[] {\n  validateEvents(input.events);\n\n  const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } =\n    prepareRecorderGenerationContext(input);\n  const screenshotAssets =\n    limitScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);\n  const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);\n  return createRecorderMarkdownReplayPromptFromContext(\n    input,\n    summary,\n    screenshotAssets,\n  );\n}\n\nasync function createRecorderMarkdownReplayPromptForGeneration(\n  input: RecorderMarkdownGenerationInput,\n): Promise<ChatCompletionMessageParam[]> {\n  validateEvents(input.events);\n\n  const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } =\n    prepareRecorderGenerationContext(input);\n  const screenshotAssets =\n    await prepareScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);\n  const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);\n  const prompt = createRecorderMarkdownReplayPromptFromContext(\n    input,\n    summary,\n    screenshotAssets,\n  );\n  debugMarkdownReplay('markdown replay prompt shape %o', {\n    eventCount: input.events.length,\n    maxScreenshots: input.maxScreenshots,\n    rawScreenshots: summarizeScreenshotAssets(rawScreenshotAssets),\n    includedScreenshots: summarizeScreenshotAssets(screenshotAssets),\n    prompt: getPromptShape(prompt),\n  });\n  return prompt;\n}\n\nfunction createRecorderMarkdownReplayPromptFromContext(\n  input: RecorderMarkdownGenerationInput,\n  summary: ReturnType<typeof prepareRecorderGenerationContext>['summary'],\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n): ChatCompletionMessageParam[] {\n  const screenshotIndexByEventHash = new Map(\n    screenshotAssets.map((asset, index) => [\n      asset.eventHashId,\n      `screenshot-${index + 1}`,\n    ]),\n  );\n  const events = summary.events.map((event) => {\n    const screenshotRef = screenshotIndexByEventHash.get(event.hashId);\n    const { screenshotPath, ...eventWithoutScreenshotPath } = event;\n    return screenshotRef\n      ? { ...eventWithoutScreenshotPath, screenshotRef }\n      : eventWithoutScreenshotPath;\n  });\n  const promptPayload = {\n    testName: input.testName || summary.testName,\n    target: {\n      platformId: input.target.platformId,\n      label: input.target.label,\n      values: input.target.values,\n    },\n    startUrl: summary.startUrl,\n    events,\n    screenshots: screenshotAssets.map((asset, index) => ({\n      screenshotRef: `screenshot-${index + 1}`,\n      eventIndex: asset.eventIndex,\n      eventHashId: asset.eventHashId,\n      eventType: asset.eventType,\n      description: getMidsceneRecorderEventDescription(\n        input.events[asset.eventIndex],\n      ),\n    })),\n  };\n  const promptText = `Generate a Markdown replay script for Midscene Agent. It will be executed with:\nawait agent.aiAct(markdownReplayPrompt)\n\nUse only the recorder data and screenshots below.\n\nTarget block:\n${stringifyMidsceneRecorderTargetBlock(input.target)}\n\nReplay goal:\n- Reproduce the recorded user workflow exactly.\n- Preserve event order.\n- Preserve the user's original intent.\n- Do not invent alternative navigation paths.\n- Do not skip, merge, reorder, or add extra user actions.\n- Prefer recorded UI text, element descriptions, URLs, input values, and scroll direction.\n- For input events, enter event.typedText/event.value exactly; do not infer or correct the text from screenshots.\n- Prefer event.semantic.replayInstruction and event.semantic.elementDescription when event.semantic.source is \"aiDescribe\" or \"recorderAI\" and event.semantic.status is \"ready\".\n- For scroll events, preserve the recorded scroll region from event.semantic.elementDescription/replayInstruction. If the scroll happened in a specific panel, list, table, dialog body, menu, navigation area, or content pane, keep that region in the Markdown step instead of generalizing it to the whole page.\n- If event.semantic.source is \"heuristic\" or event.semantic.status is \"pending\"/\"failed\", use the screenshot/context to write the best visual instruction.\n- Coordinates are only fallback hints. Do not make coordinates the primary instruction when text or screenshots are available.\n- For a click/tap that only focuses a field before an input event, describe the target as the field/control itself. Do not target a placeholder character, typed character, caret, or inner text fragment inside the field.\n- If a target cannot be found, stop and report the missing step. Do not click similar-looking elements.\n- Screenshots are only generation-time visual evidence for you. The generated Markdown will be passed directly to agent.aiAct(markdownReplayPrompt), which accepts text only and cannot receive attached images.\n- Convert any useful screenshot evidence into textual replay instructions. Do not include screenshots, image syntax, image paths, or reference-image names in the generated Markdown.\n- Never write Markdown image syntax such as ![step context](...), reference-style images, HTML <img> tags, ./screenshots/... paths, or screenshot-* names in the output.\n\nRequired structure:\n# ${input.testName || summary.testName}\n\n## Goal\nReproduce the recorded user workflow exactly.\n\n## Target\n- Platform: ${input.target.platformId}\n- Start target: ${summary.startUrl || input.target.label || input.target.deviceId || 'Recorded target'}\n\n## Steps\n1. ...\n\nRecorder data:\n${JSON.stringify(promptPayload, null, 2)}${getMarkdownLanguageInstruction(input.language)}\n\nImportant: Return ONLY raw Markdown. Do NOT wrap the response in markdown code blocks.`;\n\n  const content: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  for (const asset of screenshotAssets) {\n    const screenshotRef = screenshotIndexByEventHash.get(asset.eventHashId);\n    content.push({\n      type: 'text',\n      text: `${screenshotRef} for event #${asset.eventIndex + 1}`,\n    });\n    content.push({\n      type: 'image_url',\n      image_url: {\n        url: asset.dataUrl,\n      },\n    });\n  }\n\n  return [\n    {\n      role: 'system',\n      content:\n        'You generate precise Markdown replay scripts for Midscene agent.aiAct. The final output is plain text that will be passed directly to agent.aiAct, so it must be deterministic, ordered, safe for AI execution, and must not contain image references, screenshot paths, or screenshot labels.',\n    },\n    {\n      role: 'user',\n      content,\n    },\n  ];\n}\n\nexport async function generateRecorderMarkdownReplay(\n  input: RecorderMarkdownGenerationInput,\n  model: IModelConfig | ModelRuntime,\n): Promise<string> {\n  try {\n    const prompt = await createRecorderMarkdownReplayPromptForGeneration(input);\n    const response = await callAIWithStringResponse(\n      prompt,\n      resolveModelRuntime(model),\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return normalizeGeneratedMarkdown(response.content);\n    }\n\n    throw new Error('Failed to generate recorder Markdown replay');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder Markdown replay: ${error}`);\n  }\n}\n\nexport async function convertRecordLogIntoMarkdown(\n  log: RecorderMarkdownGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> {\n  return generateRecorderMarkdownReplay(log, modelConfig);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET","MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE","debugMarkdownReplay","getDebug","limitScreenshotAssetsForMarkdownReplay","screenshotAssets","usedPayload","asset","payloadSize","compressScreenshotAssetForMarkdownReplay","width","height","imageInfoOfBase64","longestEdge","Math","scale","dataUrl","resizeImgBase64","body","mimeType","parseBase64","prepareScreenshotAssetsForMarkdownReplay","compressedAssets","summarizeScreenshotAssets","payloadSizes","sum","size","getPromptShape","prompt","textChars","imageCount","message","content","Array","part","String","removeOmittedScreenshotPaths","summary","includedScreenshotPaths","Set","event","undefined","getMarkdownLanguageInstruction","language","normalizedLanguage","normalizeGeneratedMarkdown","trimmed","fencedMatch","resolveModelRuntime","model","getModelRuntime","createRecorderMarkdownReplayPrompt","input","validateEvents","rawSummary","rawScreenshotAssets","prepareRecorderGenerationContext","createRecorderMarkdownReplayPromptFromContext","createRecorderMarkdownReplayPromptForGeneration","screenshotIndexByEventHash","Map","index","events","screenshotRef","screenshotPath","eventWithoutScreenshotPath","promptPayload","getMidsceneRecorderEventDescription","promptText","stringifyMidsceneRecorderTargetBlock","JSON","generateRecorderMarkdownReplay","response","callAIWithStringResponse","Error","error","convertRecordLogIntoMarkdown","log","modelConfig"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACiBA,MAAMI,4CAA4C;AAClD,MAAMC,sCAAsC;AAC5C,MAAMC,sBAAsBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,wBAAwB;IAC3D,SAAS;AACX;AAEA,SAASC,uCACPC,gBAA2D;IAE3D,IAAIC,cAAc;IAClB,OAAOD,iBAAiB,MAAM,CAAC,CAACE;QAC9B,MAAMC,cAAcD,MAAM,OAAO,CAAC,MAAM;QACxC,IACEC,cAAcR,6CACdM,cAAcE,cAAcR,2CAE5B,OAAO;QAETM,eAAeE;QACf,OAAO;IACT;AACF;AAEA,eAAeC,yCACbF,KAA8C;IAE9C,MAAM,EAAEG,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMC,AAAAA,IAAAA,oBAAAA,iBAAAA,AAAAA,EAAkBL,MAAM,OAAO;IAC/D,MAAMM,cAAcC,KAAK,GAAG,CAACJ,OAAOC;IACpC,IAAIE,eAAeZ,qCACjB,OAAOM;IAGT,MAAMQ,QAAQd,sCAAsCY;IACpD,MAAMG,UAAU,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBV,MAAM,OAAO,EAAE;QACnD,OAAOO,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACJ,QAAQK;QACtC,QAAQD,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACH,SAASI;IAC1C;IACA,MAAM,EAAEG,IAAI,EAAEC,QAAQ,EAAE,GAAGC,AAAAA,IAAAA,oBAAAA,WAAAA,AAAAA,EAAYJ;IACvC,OAAO;QACL,GAAGT,KAAK;QACRS;QACA,YAAYE;QACZC;IACF;AACF;AAEA,eAAeE,yCACbhB,gBAA2D;IAE3D,MAAMiB,mBAA8D,EAAE;IACtE,KAAK,MAAMf,SAASF,iBAClB,IAAI;QACFiB,iBAAiB,IAAI,CACnB,MAAMb,yCAAyCF;IAEnD,EAAE,OAAM;QACNe,iBAAiB,IAAI,CAACf;IACxB;IAEF,OAAOH,uCAAuCkB;AAChD;AAEA,SAASC,0BACPlB,gBAA2D;IAE3D,MAAMmB,eAAenB,iBAAiB,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO,CAAC,MAAM;IACzE,OAAO;QACL,OAAOF,iBAAiB,MAAM;QAC9B,mBAAmBmB,aAAa,MAAM,CAAC,CAACC,KAAKC,OAASD,MAAMC,MAAM;QAClE,iBAAiBF,aAAa,MAAM,GAAGV,KAAK,GAAG,IAAIU,gBAAgB;IACrE;AACF;AAEA,SAASG,eAAeC,MAAoC;IAC1D,IAAIC,YAAY;IAChB,IAAIC,aAAa;IACjB,KAAK,MAAMC,WAAWH,OAAQ;QAC5B,MAAMI,UAAUD,QAAQ,OAAO;QAC/B,IAAI,AAAmB,YAAnB,OAAOC,SAAsB;YAC/BH,aAAaG,QAAQ,MAAM;YAC3B;QACF;QACA,IAAKC,MAAM,OAAO,CAACD,UAGnB;YAAA,KAAK,MAAME,QAAQF,QACjB,IAAI,AAAgB,YAAhB,OAAOE,QAAqBA,QAAQ,UAAUA,MAAM;gBACtD,IAAIA,AAAc,WAAdA,KAAK,IAAI,IAAe,UAAUA,MACpCL,aAAaM,OAAOD,KAAK,IAAI,EAAE,MAAM;gBAEvC,IAAIA,AAAc,gBAAdA,KAAK,IAAI,EACXJ,cAAc;YAElB;QACF;IACF;IACA,OAAO;QAAED;QAAWC;IAAW;AACjC;AAEA,SAASM,6BACPC,OAAuE,EACvEhC,gBAA2D;IAE3D,MAAMiC,0BAA0B,IAAIC,IAClClC,iBAAiB,GAAG,CAAC,CAACE,QAAUA,MAAM,YAAY;IAEpD,OAAO;QACL,GAAG8B,OAAO;QACV,QAAQA,QAAQ,MAAM,CAAC,GAAG,CAAC,CAACG,QAC1BA,MAAM,cAAc,IAAI,CAACF,wBAAwB,GAAG,CAACE,MAAM,cAAc,IACrE;gBAAE,GAAGA,KAAK;gBAAE,gBAAgBC;YAAU,IACtCD;IAER;AACF;AAEA,SAASE,+BAA+BC,QAAiB;IACvD,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;oDAE0C,EAAEA,mBAAmB;+EACM,CAAC;AAChF;AAEA,SAASC,2BAA2Bb,OAAe;IACjD,MAAMc,UAAUd,QAAQ,IAAI;IAC5B,MAAMe,cAAcD,QAAQ,KAAK,CAC/B;IAEF,OAAO,GAAIC,AAAAA,CAAAA,aAAa,CAAC,EAAE,IAAID,OAAM,EAAG,IAAI,GAAG,EAAE,CAAC;AACpD;AAEA,SAASE,oBAAoBC,KAAkC;IAC7D,IAAI,YAAYA,SAAS,aAAaA,OACpC,OAAOA;IAET,OAAOC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgBD;AACzB;AAEO,SAASE,mCACdC,KAAsC;IAEtCC,IAAAA,uDAAAA,cAAAA,AAAAA,EAAeD,MAAM,MAAM;IAE3B,MAAM,EAAE,SAASE,UAAU,EAAE,kBAAkBC,mBAAmB,EAAE,GAClEC,AAAAA,IAAAA,uDAAAA,gCAAAA,AAAAA,EAAiCJ;IACnC,MAAM/C,mBACJD,uCAAuCmD;IACzC,MAAMlB,UAAUD,6BAA6BkB,YAAYjD;IACzD,OAAOoD,8CACLL,OACAf,SACAhC;AAEJ;AAEA,eAAeqD,gDACbN,KAAsC;IAEtCC,IAAAA,uDAAAA,cAAAA,AAAAA,EAAeD,MAAM,MAAM;IAE3B,MAAM,EAAE,SAASE,UAAU,EAAE,kBAAkBC,mBAAmB,EAAE,GAClEC,AAAAA,IAAAA,uDAAAA,gCAAAA,AAAAA,EAAiCJ;IACnC,MAAM/C,mBACJ,MAAMgB,yCAAyCkC;IACjD,MAAMlB,UAAUD,6BAA6BkB,YAAYjD;IACzD,MAAMuB,SAAS6B,8CACbL,OACAf,SACAhC;IAEFH,oBAAoB,mCAAmC;QACrD,YAAYkD,MAAM,MAAM,CAAC,MAAM;QAC/B,gBAAgBA,MAAM,cAAc;QACpC,gBAAgB7B,0BAA0BgC;QAC1C,qBAAqBhC,0BAA0BlB;QAC/C,QAAQsB,eAAeC;IACzB;IACA,OAAOA;AACT;AAEA,SAAS6B,8CACPL,KAAsC,EACtCf,OAAuE,EACvEhC,gBAA2D;IAE3D,MAAMsD,6BAA6B,IAAIC,IACrCvD,iBAAiB,GAAG,CAAC,CAACE,OAAOsD,QAAU;YACrCtD,MAAM,WAAW;YACjB,CAAC,WAAW,EAAEsD,QAAQ,GAAG;SAC1B;IAEH,MAAMC,SAASzB,QAAQ,MAAM,CAAC,GAAG,CAAC,CAACG;QACjC,MAAMuB,gBAAgBJ,2BAA2B,GAAG,CAACnB,MAAM,MAAM;QACjE,MAAM,EAAEwB,cAAc,EAAE,GAAGC,4BAA4B,GAAGzB;QAC1D,OAAOuB,gBACH;YAAE,GAAGE,0BAA0B;YAAEF;QAAc,IAC/CE;IACN;IACA,MAAMC,gBAAgB;QACpB,UAAUd,MAAM,QAAQ,IAAIf,QAAQ,QAAQ;QAC5C,QAAQ;YACN,YAAYe,MAAM,MAAM,CAAC,UAAU;YACnC,OAAOA,MAAM,MAAM,CAAC,KAAK;YACzB,QAAQA,MAAM,MAAM,CAAC,MAAM;QAC7B;QACA,UAAUf,QAAQ,QAAQ;QAC1ByB;QACA,aAAazD,iBAAiB,GAAG,CAAC,CAACE,OAAOsD,QAAW;gBACnD,eAAe,CAAC,WAAW,EAAEA,QAAQ,GAAG;gBACxC,YAAYtD,MAAM,UAAU;gBAC5B,aAAaA,MAAM,WAAW;gBAC9B,WAAWA,MAAM,SAAS;gBAC1B,aAAa4D,AAAAA,IAAAA,yBAAAA,mCAAAA,AAAAA,EACXf,MAAM,MAAM,CAAC7C,MAAM,UAAU,CAAC;YAElC;IACF;IACA,MAAM6D,aAAa,CAAC;;;;;;AAMtB,EAAEC,AAAAA,IAAAA,yBAAAA,oCAAAA,AAAAA,EAAqCjB,MAAM,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;EAqBnD,EAAEA,MAAM,QAAQ,IAAIf,QAAQ,QAAQ,CAAC;;;;;;YAM3B,EAAEe,MAAM,MAAM,CAAC,UAAU,CAAC;gBACtB,EAAEf,QAAQ,QAAQ,IAAIe,MAAM,MAAM,CAAC,KAAK,IAAIA,MAAM,MAAM,CAAC,QAAQ,IAAI,kBAAkB;;;;;;AAMvG,EAAEkB,KAAK,SAAS,CAACJ,eAAe,MAAM,KAAKxB,+BAA+BU,MAAM,QAAQ,EAAE;;sFAEJ,CAAC;IAErF,MAAMpB,UAAiB;QACrB;YACE,MAAM;YACN,MAAMoC;QACR;KACD;IAED,KAAK,MAAM7D,SAASF,iBAAkB;QACpC,MAAM0D,gBAAgBJ,2BAA2B,GAAG,CAACpD,MAAM,WAAW;QACtEyB,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,GAAG+B,cAAc,YAAY,EAAExD,MAAM,UAAU,GAAG,GAAG;QAC7D;QACAyB,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,WAAW;gBACT,KAAKzB,MAAM,OAAO;YACpB;QACF;IACF;IAEA,OAAO;QACL;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACNyB;QACF;KACD;AACH;AAEO,eAAeuC,+BACpBnB,KAAsC,EACtCH,KAAkC;IAElC,IAAI;QACF,MAAMrB,SAAS,MAAM8B,gDAAgDN;QACrE,MAAMoB,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACrB7C,QACAoB,oBAAoBC;QAGtB,IAAIuB,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO3B,2BAA2B2B,SAAS,OAAO;QAGpD,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOC,OAAO;QACd,MAAM,IAAID,MAAM,CAAC,6CAA6C,EAAEC,OAAO;IACzE;AACF;AAEO,eAAeC,6BACpBC,GAAoC,EACpCC,WAAyB;IAEzB,OAAOP,+BAA+BM,KAAKC;AAC7C"}

package/dist/lib/ai-model/prompt/recorder-generation-common.js

@@ -32,12 +32,48 @@ __webpack_require__.d(__webpack_exports__, {
     extractInputDescriptions: ()=>extractInputDescriptions,
     prepareEventSummary: ()=>prepareEventSummary,
     prepareRecorderGenerationContext: ()=>prepareRecorderGenerationContext,
+    compactRecorderSemanticForGeneration: ()=>compactRecorderSemanticForGeneration,
     processEventsForLLM: ()=>processEventsForLLM
 });
 const recorder_namespaceObject = require("@midscene/shared/recorder");
+const MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH = 1200;
+const MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH = 400;
 function cleanRecorderSemanticField(value) {
     return value?.trim() === 'AI is analyzing element...' ? void 0 : value;
 }
+function truncateRecorderGenerationText(value, maxLength) {
+    if (value.length <= maxLength) return value;
+    return `${value.slice(0, maxLength)}... [truncated ${value.length - maxLength} chars]`;
+}
+function compactRecorderSemanticText(value) {
+    const cleaned = cleanRecorderSemanticField(value);
+    return cleaned ? truncateRecorderGenerationText(cleaned, MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH) : void 0;
+}
+function compactRecorderSemanticError(value) {
+    return value ? truncateRecorderGenerationText(value, MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH) : void 0;
+}
+function compactRecorderSemanticForGeneration(semantic) {
+    if (!semantic) return;
+    return {
+        source: semantic.source,
+        status: semantic.status,
+        confidence: semantic.confidence,
+        elementDescription: compactRecorderSemanticText(semantic.elementDescription),
+        replayInstruction: compactRecorderSemanticText(semantic.replayInstruction),
+        actionSummary: compactRecorderSemanticText(semantic.actionSummary),
+        error: compactRecorderSemanticError(semantic.error),
+        ...semantic.aiDescribe ? {
+            aiDescribe: {
+                verifyPrompt: semantic.aiDescribe.verifyPrompt,
+                verifyPassed: semantic.aiDescribe.verifyPassed,
+                deepLocate: semantic.aiDescribe.deepLocate,
+                centerDistance: semantic.aiDescribe.centerDistance,
+                expectedCenter: semantic.aiDescribe.expectedCenter,
+                actualCenter: semantic.aiDescribe.actualCenter
+            }
+        } : {}
+    };
+}
 const validateEvents = (events)=>{
     if (!events.length) throw new Error('No events provided for test generation');
 };
@@ -58,35 +94,60 @@ const createEventCounts = (filteredEvents, totalEvents)=>({
         scroll: filteredEvents.scrollEvents.length,
         total: totalEvents
     });
-const extractInputDescriptions = (inputEvents)=>inputEvents.map((event)=>({
-            description: cleanRecorderSemanticField(event.elementDescription) || '',
+const extractInputDescriptions = (inputEvents)=>inputEvents.map((event)=>{
+        const semantic = (0, recorder_namespaceObject.getMidsceneRecorderSemantic)(event);
+        return {
+            description: cleanRecorderSemanticField(semantic?.elementDescription) || '',
             value: event.value || ''
-        })).filter((item)=>item.description && item.value);
-const processEventsForLLM = (events, screenshotPathByEventHash = new Map())=>events.map((event)=>({
+        };
+    }).filter((item)=>item.description && item.value);
+const processEventsForLLM = (events, screenshotPathByEventHash = new Map())=>{
+    let inputIndex = 0;
+    return events.map((event, index)=>{
+        const previousEvent = events[index - 1];
+        const nextEvent = events[index + 1];
+        const previousInput = events.slice(0, index).reverse().find((candidate)=>'input' === candidate.type);
+        const nextInput = events.slice(index + 1).find((candidate)=>'input' === candidate.type);
+        const isInput = 'input' === event.type;
+        const inputSequenceIndex = isInput ? ++inputIndex : void 0;
+        const hasNeighborInput = Boolean(previousInput || nextInput);
+        const neighborInputValues = isInput ? [
+            previousInput?.value,
+            nextInput?.value
+        ].filter((value)=>Boolean(value)) : void 0;
+        const semantic = compactRecorderSemanticForGeneration((0, recorder_namespaceObject.getMidsceneRecorderSemantic)(event));
+        return {
             hashId: event.hashId,
             type: event.type,
             timestamp: event.timestamp,
             source: event.source,
             actionType: event.actionType,
-            descriptionSource: event.descriptionSource,
-            descriptionError: event.descriptionError,
             url: event.url,
             title: event.title,
-            elementDescription: cleanRecorderSemanticField(event.elementDescription),
-            replayInstruction: cleanRecorderSemanticField(event.replayInstruction),
-            actionSummary: cleanRecorderSemanticField(event.actionSummary),
-            semanticConfidence: event.semanticConfidence,
+            semantic,
             description: (0, recorder_namespaceObject.getMidsceneRecorderEventDescription)(event),
             value: event.value,
+            previousActionDescription: previousEvent ? (0, recorder_namespaceObject.getMidsceneRecorderEventDescription)(previousEvent) : void 0,
+            nextActionDescription: nextEvent ? (0, recorder_namespaceObject.getMidsceneRecorderEventDescription)(nextEvent) : void 0,
+            ...isInput ? {
+                typedText: event.value || '',
+                inputIndex: inputSequenceIndex,
+                isSequentialInput: previousEvent?.type === 'input' || nextEvent?.type === 'input',
+                hasNeighborInput,
+                previousInputDescription: previousInput ? (0, recorder_namespaceObject.getMidsceneRecorderEventDescription)(previousInput) : void 0,
+                neighborInputValues: neighborInputValues && neighborInputValues.length > 0 ? neighborInputValues : void 0
+            } : {},
             pageInfo: event.pageInfo,
             elementRect: event.elementRect,
             screenshotPath: screenshotPathByEventHash.get(event.hashId)
-        }));
+        };
+    });
+};
 const prepareEventSummary = (events, options = {})=>{
     const filteredEvents = filterEventsByType(events);
     const eventCounts = createEventCounts(filteredEvents, events.length);
     const startUrl = filteredEvents.navigationEvents.length > 0 ? filteredEvents.navigationEvents[0].url || '' : '';
-    const clickDescriptions = filteredEvents.clickEvents.map((event)=>event.elementDescription).filter((desc)=>Boolean(desc)).slice(0, 10);
+    const clickDescriptions = filteredEvents.clickEvents.map((event)=>(0, recorder_namespaceObject.getMidsceneRecorderSemantic)(event)?.elementDescription).filter((desc)=>Boolean(desc)).slice(0, 10);
     const inputDescriptions = extractInputDescriptions(filteredEvents.inputEvents).slice(0, 10);
     const urls = filteredEvents.navigationEvents.map((e)=>e.url).filter((url)=>Boolean(url)).slice(0, 5);
     const processedEvents = processEventsForLLM(events, options.screenshotPathByEventHash);
@@ -142,6 +203,7 @@ const createMessageContent = (promptText, screenshots = [], includeScreenshots =
     }
     return messageContent;
 };
+exports.compactRecorderSemanticForGeneration = __webpack_exports__.compactRecorderSemanticForGeneration;
 exports.createEventCounts = __webpack_exports__.createEventCounts;
 exports.createMessageContent = __webpack_exports__.createMessageContent;
 exports.extractInputDescriptions = __webpack_exports__.extractInputDescriptions;
@@ -152,6 +214,7 @@ exports.prepareRecorderGenerationContext = __webpack_exports__.prepareRecorderGe
 exports.processEventsForLLM = __webpack_exports__.processEventsForLLM;
 exports.validateEvents = __webpack_exports__.validateEvents;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "compactRecorderSemanticForGeneration",
     "createEventCounts",
     "createMessageContent",
     "extractInputDescriptions",

package/dist/lib/ai-model/prompt/recorder-generation-common.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/recorder-generation-common.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/recorder-generation-common.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS,\n  type MidsceneRecorderEvent,\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  type MidsceneRecorderTarget,\n  createMidsceneRecorderMarkdownScreenshotAssets,\n  getMidsceneRecorderEventDescription,\n} from '@midscene/shared/recorder';\n\nexport interface EventCounts {\n  navigation: number;\n  click: number;\n  input: number;\n  scroll: number;\n  total: number;\n}\n\nexport interface InputDescription {\n  description: string;\n  value: string;\n}\n\nexport interface ProcessedEvent {\n  hashId: string;\n  type: string;\n  timestamp: number;\n  source?: string;\n  actionType?: string;\n  descriptionSource?: string;\n  descriptionError?: string;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  replayInstruction?: string;\n  actionSummary?: string;\n  semanticConfidence?: string;\n  description?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n  screenshotPath?: string;\n}\n\nexport interface EventSummary {\n  testName: string;\n  startUrl: string;\n  eventCounts: EventCounts;\n  urls: string[];\n  clickDescriptions: string[];\n  inputDescriptions: InputDescription[];\n  events: ProcessedEvent[];\n}\n\nexport interface RecorderGenerationContext {\n  summary: EventSummary;\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n}\n\nexport type ChromeRecordedEvent = MidsceneRecorderEvent;\n\nexport interface RecorderGenerationOptions {\n  testName?: string;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  /** Language for human-readable generated content (e.g. 'English', 'Chinese'). Keys and API names are kept as-is. */\n  language?: string;\n  navigationInfo?: {\n    urls?: string[];\n    titles?: string[];\n    initialViewport?: {\n      width?: number;\n      height?: number;\n    };\n  };\n}\n\nexport interface RecorderGenerationInput extends RecorderGenerationOptions {\n  target: MidsceneRecorderTarget;\n  events: MidsceneRecorderEvent[];\n}\n\nexport interface FilteredEvents {\n  navigationEvents: ChromeRecordedEvent[];\n  clickEvents: ChromeRecordedEvent[];\n  inputEvents: ChromeRecordedEvent[];\n  scrollEvents: ChromeRecordedEvent[];\n}\n\nfunction cleanRecorderSemanticField(value?: string) {\n  return value?.trim() === 'AI is analyzing element...' ? undefined : value;\n}\n\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n  if (!events.length) {\n    throw new Error('No events provided for test generation');\n  }\n};\n\nexport const getScreenshotsForLLM = (\n  events: ChromeRecordedEvent[],\n  maxScreenshots = 1,\n): string[] => {\n  return createMidsceneRecorderMarkdownScreenshotAssets(events, {\n    baseDir: './screenshots',\n    maxScreenshots,\n  }).map((asset) => asset.dataUrl);\n};\n\nexport const filterEventsByType = (\n  events: ChromeRecordedEvent[],\n): FilteredEvents => {\n  return {\n    navigationEvents: events.filter((event) => event.type === 'navigation'),\n    clickEvents: events.filter((event) => event.type === 'click'),\n    inputEvents: events.filter((event) => event.type === 'input'),\n    scrollEvents: events.filter((event) => event.type === 'scroll'),\n  };\n};\n\nexport const createEventCounts = (\n  filteredEvents: FilteredEvents,\n  totalEvents: number,\n): EventCounts => {\n  return {\n    navigation: filteredEvents.navigationEvents.length,\n    click: filteredEvents.clickEvents.length,\n    input: filteredEvents.inputEvents.length,\n    scroll: filteredEvents.scrollEvents.length,\n    total: totalEvents,\n  };\n};\n\nexport const extractInputDescriptions = (\n  inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n  return inputEvents\n    .map((event) => ({\n      description: cleanRecorderSemanticField(event.elementDescription) || '',\n      value: event.value || '',\n    }))\n    .filter((item) => item.description && item.value);\n};\n\nexport const processEventsForLLM = (\n  events: ChromeRecordedEvent[],\n  screenshotPathByEventHash: Map<string, string> = new Map(),\n): ProcessedEvent[] => {\n  return events.map((event) => ({\n    hashId: event.hashId,\n    type: event.type,\n    timestamp: event.timestamp,\n    source: event.source,\n    actionType: event.actionType,\n    descriptionSource: event.descriptionSource,\n    descriptionError: event.descriptionError,\n    url: event.url,\n    title: event.title,\n    elementDescription: cleanRecorderSemanticField(event.elementDescription),\n    replayInstruction: cleanRecorderSemanticField(event.replayInstruction),\n    actionSummary: cleanRecorderSemanticField(event.actionSummary),\n    semanticConfidence: event.semanticConfidence,\n    description: getMidsceneRecorderEventDescription(event),\n    value: event.value,\n    pageInfo: event.pageInfo,\n    elementRect: event.elementRect,\n    screenshotPath: screenshotPathByEventHash.get(event.hashId),\n  }));\n};\n\nexport const prepareEventSummary = (\n  events: ChromeRecordedEvent[],\n  options: {\n    testName?: string;\n    maxScreenshots?: number;\n    screenshotPathByEventHash?: Map<string, string>;\n  } = {},\n): EventSummary => {\n  const filteredEvents = filterEventsByType(events);\n  const eventCounts = createEventCounts(filteredEvents, events.length);\n\n  const startUrl =\n    filteredEvents.navigationEvents.length > 0\n      ? filteredEvents.navigationEvents[0].url || ''\n      : '';\n\n  const clickDescriptions = filteredEvents.clickEvents\n    .map((event) => event.elementDescription)\n    .filter((desc): desc is string => Boolean(desc))\n    .slice(0, 10);\n\n  const inputDescriptions = extractInputDescriptions(\n    filteredEvents.inputEvents,\n  ).slice(0, 10);\n\n  const urls = filteredEvents.navigationEvents\n    .map((e) => e.url)\n    .filter((url): url is string => Boolean(url))\n    .slice(0, 5);\n\n  const processedEvents = processEventsForLLM(\n    events,\n    options.screenshotPathByEventHash,\n  );\n\n  return {\n    testName: options.testName || 'Automated test from recorded events',\n    startUrl,\n    eventCounts,\n    urls,\n    clickDescriptions,\n    inputDescriptions,\n    events: processedEvents,\n  };\n};\n\nexport function prepareRecorderGenerationContext(\n  input: RecorderGenerationInput,\n): RecorderGenerationContext {\n  validateEvents(input.events);\n\n  const maxScreenshots =\n    input.maxScreenshots ?? DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS;\n  const screenshotAssets = createMidsceneRecorderMarkdownScreenshotAssets(\n    input.events,\n    {\n      baseDir: './screenshots',\n      maxScreenshots,\n    },\n  );\n  const screenshotPathByEventHash = new Map(\n    screenshotAssets.map((asset) => [asset.eventHashId, asset.relativePath]),\n  );\n\n  return {\n    summary: prepareEventSummary(input.events, {\n      testName: input.testName,\n      screenshotPathByEventHash,\n    }),\n    screenshotAssets,\n  };\n}\n\nexport const createMessageContent = (\n  promptText: string,\n  screenshots: string[] = [],\n  includeScreenshots = true,\n) => {\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  if (includeScreenshots && screenshots.length > 0) {\n    messageContent.unshift({\n      type: 'text',\n      text: 'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    screenshots.forEach((screenshot) => {\n      messageContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      });\n    });\n  }\n\n  return messageContent;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","cleanRecorderSemanticField","value","undefined","validateEvents","events","Error","getScreenshotsForLLM","maxScreenshots","createMidsceneRecorderMarkdownScreenshotAssets","asset","filterEventsByType","event","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","item","processEventsForLLM","screenshotPathByEventHash","Map","getMidsceneRecorderEventDescription","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","Boolean","inputDescriptions","urls","e","url","processedEvents","prepareRecorderGenerationContext","input","DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS","screenshotAssets","createMessageContent","promptText","screenshots","includeScreenshots","messageContent","screenshot"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;ACmFA,SAASI,2BAA2BC,KAAc;IAChD,OAAOA,OAAO,WAAW,+BAA+BC,SAAYD;AACtE;AAEO,MAAME,iBAAiB,CAACC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIC,MAAM;AAEpB;AAEO,MAAMC,uBAAuB,CAClCF,QACAG,iBAAiB,CAAC,GAEXC,AAAAA,IAAAA,yBAAAA,8CAAAA,AAAAA,EAA+CJ,QAAQ;QAC5D,SAAS;QACTG;IACF,GAAG,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO;AAG1B,MAAMC,qBAAqB,CAChCN,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAGK,MAAMC,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAGK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACL,QAAW;YACf,aAAaX,2BAA2BW,MAAM,kBAAkB,KAAK;YACrE,OAAOA,MAAM,KAAK,IAAI;QACxB,IACC,MAAM,CAAC,CAACM,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAG7C,MAAMC,sBAAsB,CACjCd,QACAe,4BAAiD,IAAIC,KAAK,GAEnDhB,OAAO,GAAG,CAAC,CAACO,QAAW;YAC5B,QAAQA,MAAM,MAAM;YACpB,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,QAAQA,MAAM,MAAM;YACpB,YAAYA,MAAM,UAAU;YAC5B,mBAAmBA,MAAM,iBAAiB;YAC1C,kBAAkBA,MAAM,gBAAgB;YACxC,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClB,oBAAoBX,2BAA2BW,MAAM,kBAAkB;YACvE,mBAAmBX,2BAA2BW,MAAM,iBAAiB;YACrE,eAAeX,2BAA2BW,MAAM,aAAa;YAC7D,oBAAoBA,MAAM,kBAAkB;YAC5C,aAAaU,AAAAA,IAAAA,yBAAAA,mCAAAA,AAAAA,EAAoCV;YACjD,OAAOA,MAAM,KAAK;YAClB,UAAUA,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;YAC9B,gBAAgBQ,0BAA0B,GAAG,CAACR,MAAM,MAAM;QAC5D;AAGK,MAAMW,sBAAsB,CACjClB,QACAmB,UAII,CAAC,CAAC;IAEN,MAAMV,iBAAiBH,mBAAmBN;IAC1C,MAAMoB,cAAcZ,kBAAkBC,gBAAgBT,OAAO,MAAM;IAEnE,MAAMqB,WACJZ,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAMa,oBAAoBb,eAAe,WAAW,CACjD,GAAG,CAAC,CAACF,QAAUA,MAAM,kBAAkB,EACvC,MAAM,CAAC,CAACgB,OAAyBC,QAAQD,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAME,oBAAoBd,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAMiB,OAAOjB,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAACkB,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBJ,QAAQI,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkBf,oBACtBd,QACAmB,QAAQ,yBAAyB;IAGnC,OAAO;QACL,UAAUA,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAM;QACAJ;QACAG;QACA,QAAQI;IACV;AACF;AAEO,SAASC,iCACdC,KAA8B;IAE9BhC,eAAegC,MAAM,MAAM;IAE3B,MAAM5B,iBACJ4B,MAAM,cAAc,IAAIC,yBAAAA,kDAAkDA;IAC5E,MAAMC,mBAAmB7B,AAAAA,IAAAA,yBAAAA,8CAAAA,AAAAA,EACvB2B,MAAM,MAAM,EACZ;QACE,SAAS;QACT5B;IACF;IAEF,MAAMY,4BAA4B,IAAIC,IACpCiB,iBAAiB,GAAG,CAAC,CAAC5B,QAAU;YAACA,MAAM,WAAW;YAAEA,MAAM,YAAY;SAAC;IAGzE,OAAO;QACL,SAASa,oBAAoBa,MAAM,MAAM,EAAE;YACzC,UAAUA,MAAM,QAAQ;YACxBhB;QACF;QACAkB;IACF;AACF;AAEO,MAAMC,uBAAuB,CAClCC,YACAC,cAAwB,EAAE,EAC1BC,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMH;QACR;KACD;IAED,IAAIE,sBAAsBD,YAAY,MAAM,GAAG,GAAG;QAChDE,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAF,YAAY,OAAO,CAAC,CAACG;YACnBD,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKC;gBACP;YACF;QACF;IACF;IAEA,OAAOD;AACT"}
+{"version":3,"file":"ai-model/prompt/recorder-generation-common.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/recorder-generation-common.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS,\n  type MidsceneRecorderEvent,\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  type MidsceneRecorderSemantic,\n  type MidsceneRecorderTarget,\n  createMidsceneRecorderMarkdownScreenshotAssets,\n  getMidsceneRecorderEventDescription,\n  getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\n\nexport interface EventCounts {\n  navigation: number;\n  click: number;\n  input: number;\n  scroll: number;\n  total: number;\n}\n\nexport interface InputDescription {\n  description: string;\n  value: string;\n}\n\nexport interface ProcessedEvent {\n  hashId: string;\n  type: string;\n  timestamp: number;\n  source?: string;\n  actionType?: string;\n  url?: string;\n  title?: string;\n  semantic?: MidsceneRecorderSemantic;\n  description?: string;\n  value?: string;\n  typedText?: string;\n  inputIndex?: number;\n  isSequentialInput?: boolean;\n  hasNeighborInput?: boolean;\n  previousInputDescription?: string;\n  previousActionDescription?: string;\n  nextActionDescription?: string;\n  neighborInputValues?: string[];\n  pageInfo?: any;\n  elementRect?: any;\n  screenshotPath?: string;\n}\n\nexport interface EventSummary {\n  testName: string;\n  startUrl: string;\n  eventCounts: EventCounts;\n  urls: string[];\n  clickDescriptions: string[];\n  inputDescriptions: InputDescription[];\n  events: ProcessedEvent[];\n}\n\nexport interface RecorderGenerationContext {\n  summary: EventSummary;\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n}\n\nexport type ChromeRecordedEvent = MidsceneRecorderEvent;\n\nconst MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH = 1200;\nconst MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH = 400;\n\nexport interface RecorderGenerationOptions {\n  testName?: string;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  /** Language for human-readable generated content (e.g. 'English', 'Chinese'). Keys and API names are kept as-is. */\n  language?: string;\n  navigationInfo?: {\n    urls?: string[];\n    titles?: string[];\n    initialViewport?: {\n      width?: number;\n      height?: number;\n    };\n  };\n}\n\nexport interface RecorderGenerationInput extends RecorderGenerationOptions {\n  target: MidsceneRecorderTarget;\n  events: MidsceneRecorderEvent[];\n}\n\nexport interface FilteredEvents {\n  navigationEvents: ChromeRecordedEvent[];\n  clickEvents: ChromeRecordedEvent[];\n  inputEvents: ChromeRecordedEvent[];\n  scrollEvents: ChromeRecordedEvent[];\n}\n\nfunction cleanRecorderSemanticField(value?: string) {\n  return value?.trim() === 'AI is analyzing element...' ? undefined : value;\n}\n\nfunction truncateRecorderGenerationText(value: string, maxLength: number) {\n  if (value.length <= maxLength) {\n    return value;\n  }\n  return `${value.slice(0, maxLength)}... [truncated ${value.length - maxLength} chars]`;\n}\n\nfunction compactRecorderSemanticText(value?: string) {\n  const cleaned = cleanRecorderSemanticField(value);\n  return cleaned\n    ? truncateRecorderGenerationText(\n        cleaned,\n        MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH,\n      )\n    : undefined;\n}\n\nfunction compactRecorderSemanticError(value?: string) {\n  return value\n    ? truncateRecorderGenerationText(\n        value,\n        MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH,\n      )\n    : undefined;\n}\n\nexport function compactRecorderSemanticForGeneration(\n  semantic?: MidsceneRecorderSemantic,\n): MidsceneRecorderSemantic | undefined {\n  if (!semantic) {\n    return undefined;\n  }\n\n  return {\n    source: semantic.source,\n    status: semantic.status,\n    confidence: semantic.confidence,\n    elementDescription: compactRecorderSemanticText(\n      semantic.elementDescription,\n    ),\n    replayInstruction: compactRecorderSemanticText(semantic.replayInstruction),\n    actionSummary: compactRecorderSemanticText(semantic.actionSummary),\n    error: compactRecorderSemanticError(semantic.error),\n    ...(semantic.aiDescribe\n      ? {\n          aiDescribe: {\n            verifyPrompt: semantic.aiDescribe.verifyPrompt,\n            verifyPassed: semantic.aiDescribe.verifyPassed,\n            deepLocate: semantic.aiDescribe.deepLocate,\n            centerDistance: semantic.aiDescribe.centerDistance,\n            expectedCenter: semantic.aiDescribe.expectedCenter,\n            actualCenter: semantic.aiDescribe.actualCenter,\n          },\n        }\n      : {}),\n  };\n}\n\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n  if (!events.length) {\n    throw new Error('No events provided for test generation');\n  }\n};\n\nexport const getScreenshotsForLLM = (\n  events: ChromeRecordedEvent[],\n  maxScreenshots = 1,\n): string[] => {\n  return createMidsceneRecorderMarkdownScreenshotAssets(events, {\n    baseDir: './screenshots',\n    maxScreenshots,\n  }).map((asset) => asset.dataUrl);\n};\n\nexport const filterEventsByType = (\n  events: ChromeRecordedEvent[],\n): FilteredEvents => {\n  return {\n    navigationEvents: events.filter((event) => event.type === 'navigation'),\n    clickEvents: events.filter((event) => event.type === 'click'),\n    inputEvents: events.filter((event) => event.type === 'input'),\n    scrollEvents: events.filter((event) => event.type === 'scroll'),\n  };\n};\n\nexport const createEventCounts = (\n  filteredEvents: FilteredEvents,\n  totalEvents: number,\n): EventCounts => {\n  return {\n    navigation: filteredEvents.navigationEvents.length,\n    click: filteredEvents.clickEvents.length,\n    input: filteredEvents.inputEvents.length,\n    scroll: filteredEvents.scrollEvents.length,\n    total: totalEvents,\n  };\n};\n\nexport const extractInputDescriptions = (\n  inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n  return inputEvents\n    .map((event) => {\n      const semantic = getMidsceneRecorderSemantic(event);\n      return {\n        description:\n          cleanRecorderSemanticField(semantic?.elementDescription) || '',\n        value: event.value || '',\n      };\n    })\n    .filter((item) => item.description && item.value);\n};\n\nexport const processEventsForLLM = (\n  events: ChromeRecordedEvent[],\n  screenshotPathByEventHash: Map<string, string> = new Map(),\n): ProcessedEvent[] => {\n  let inputIndex = 0;\n  return events.map((event, index) => {\n    const previousEvent = events[index - 1];\n    const nextEvent = events[index + 1];\n    const previousInput = events\n      .slice(0, index)\n      .reverse()\n      .find((candidate) => candidate.type === 'input');\n    const nextInput = events\n      .slice(index + 1)\n      .find((candidate) => candidate.type === 'input');\n    const isInput = event.type === 'input';\n    const inputSequenceIndex = isInput ? ++inputIndex : undefined;\n    const hasNeighborInput = Boolean(previousInput || nextInput);\n    const neighborInputValues = isInput\n      ? [previousInput?.value, nextInput?.value].filter(\n          (value): value is string => Boolean(value),\n        )\n      : undefined;\n    const semantic = compactRecorderSemanticForGeneration(\n      getMidsceneRecorderSemantic(event),\n    );\n\n    return {\n      hashId: event.hashId,\n      type: event.type,\n      timestamp: event.timestamp,\n      source: event.source,\n      actionType: event.actionType,\n      url: event.url,\n      title: event.title,\n      semantic,\n      description: getMidsceneRecorderEventDescription(event),\n      value: event.value,\n      previousActionDescription: previousEvent\n        ? getMidsceneRecorderEventDescription(previousEvent)\n        : undefined,\n      nextActionDescription: nextEvent\n        ? getMidsceneRecorderEventDescription(nextEvent)\n        : undefined,\n      ...(isInput\n        ? {\n            typedText: event.value || '',\n            inputIndex: inputSequenceIndex,\n            isSequentialInput:\n              previousEvent?.type === 'input' || nextEvent?.type === 'input',\n            hasNeighborInput,\n            previousInputDescription: previousInput\n              ? getMidsceneRecorderEventDescription(previousInput)\n              : undefined,\n            neighborInputValues:\n              neighborInputValues && neighborInputValues.length > 0\n                ? neighborInputValues\n                : undefined,\n          }\n        : {}),\n      pageInfo: event.pageInfo,\n      elementRect: event.elementRect,\n      screenshotPath: screenshotPathByEventHash.get(event.hashId),\n    };\n  });\n};\n\nexport const prepareEventSummary = (\n  events: ChromeRecordedEvent[],\n  options: {\n    testName?: string;\n    maxScreenshots?: number;\n    screenshotPathByEventHash?: Map<string, string>;\n  } = {},\n): EventSummary => {\n  const filteredEvents = filterEventsByType(events);\n  const eventCounts = createEventCounts(filteredEvents, events.length);\n\n  const startUrl =\n    filteredEvents.navigationEvents.length > 0\n      ? filteredEvents.navigationEvents[0].url || ''\n      : '';\n\n  const clickDescriptions = filteredEvents.clickEvents\n    .map((event) => getMidsceneRecorderSemantic(event)?.elementDescription)\n    .filter((desc): desc is string => Boolean(desc))\n    .slice(0, 10);\n\n  const inputDescriptions = extractInputDescriptions(\n    filteredEvents.inputEvents,\n  ).slice(0, 10);\n\n  const urls = filteredEvents.navigationEvents\n    .map((e) => e.url)\n    .filter((url): url is string => Boolean(url))\n    .slice(0, 5);\n\n  const processedEvents = processEventsForLLM(\n    events,\n    options.screenshotPathByEventHash,\n  );\n\n  return {\n    testName: options.testName || 'Automated test from recorded events',\n    startUrl,\n    eventCounts,\n    urls,\n    clickDescriptions,\n    inputDescriptions,\n    events: processedEvents,\n  };\n};\n\nexport function prepareRecorderGenerationContext(\n  input: RecorderGenerationInput,\n): RecorderGenerationContext {\n  validateEvents(input.events);\n\n  const maxScreenshots =\n    input.maxScreenshots ?? DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS;\n  const screenshotAssets = createMidsceneRecorderMarkdownScreenshotAssets(\n    input.events,\n    {\n      baseDir: './screenshots',\n      maxScreenshots,\n    },\n  );\n  const screenshotPathByEventHash = new Map(\n    screenshotAssets.map((asset) => [asset.eventHashId, asset.relativePath]),\n  );\n\n  return {\n    summary: prepareEventSummary(input.events, {\n      testName: input.testName,\n      screenshotPathByEventHash,\n    }),\n    screenshotAssets,\n  };\n}\n\nexport const createMessageContent = (\n  promptText: string,\n  screenshots: string[] = [],\n  includeScreenshots = true,\n) => {\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  if (includeScreenshots && screenshots.length > 0) {\n    messageContent.unshift({\n      type: 'text',\n      text: 'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    screenshots.forEach((screenshot) => {\n      messageContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      });\n    });\n  }\n\n  return messageContent;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH","MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH","cleanRecorderSemanticField","value","undefined","truncateRecorderGenerationText","maxLength","compactRecorderSemanticText","cleaned","compactRecorderSemanticError","compactRecorderSemanticForGeneration","semantic","validateEvents","events","Error","getScreenshotsForLLM","maxScreenshots","createMidsceneRecorderMarkdownScreenshotAssets","asset","filterEventsByType","event","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","getMidsceneRecorderSemantic","item","processEventsForLLM","screenshotPathByEventHash","Map","inputIndex","index","previousEvent","nextEvent","previousInput","candidate","nextInput","isInput","inputSequenceIndex","hasNeighborInput","Boolean","neighborInputValues","getMidsceneRecorderEventDescription","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","inputDescriptions","urls","e","url","processedEvents","prepareRecorderGenerationContext","input","DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS","screenshotAssets","createMessageContent","promptText","screenshots","includeScreenshots","messageContent","screenshot"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;AC2DA,MAAMI,+CAA+C;AACrD,MAAMC,gDAAgD;AA+BtD,SAASC,2BAA2BC,KAAc;IAChD,OAAOA,OAAO,WAAW,+BAA+BC,SAAYD;AACtE;AAEA,SAASE,+BAA+BF,KAAa,EAAEG,SAAiB;IACtE,IAAIH,MAAM,MAAM,IAAIG,WAClB,OAAOH;IAET,OAAO,GAAGA,MAAM,KAAK,CAAC,GAAGG,WAAW,eAAe,EAAEH,MAAM,MAAM,GAAGG,UAAU,OAAO,CAAC;AACxF;AAEA,SAASC,4BAA4BJ,KAAc;IACjD,MAAMK,UAAUN,2BAA2BC;IAC3C,OAAOK,UACHH,+BACEG,SACAR,gDAEFI;AACN;AAEA,SAASK,6BAA6BN,KAAc;IAClD,OAAOA,QACHE,+BACEF,OACAF,iDAEFG;AACN;AAEO,SAASM,qCACdC,QAAmC;IAEnC,IAAI,CAACA,UACH;IAGF,OAAO;QACL,QAAQA,SAAS,MAAM;QACvB,QAAQA,SAAS,MAAM;QACvB,YAAYA,SAAS,UAAU;QAC/B,oBAAoBJ,4BAClBI,SAAS,kBAAkB;QAE7B,mBAAmBJ,4BAA4BI,SAAS,iBAAiB;QACzE,eAAeJ,4BAA4BI,SAAS,aAAa;QACjE,OAAOF,6BAA6BE,SAAS,KAAK;QAClD,GAAIA,SAAS,UAAU,GACnB;YACE,YAAY;gBACV,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,YAAYA,SAAS,UAAU,CAAC,UAAU;gBAC1C,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,cAAcA,SAAS,UAAU,CAAC,YAAY;YAChD;QACF,IACA,CAAC,CAAC;IACR;AACF;AAEO,MAAMC,iBAAiB,CAACC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIC,MAAM;AAEpB;AAEO,MAAMC,uBAAuB,CAClCF,QACAG,iBAAiB,CAAC,GAEXC,AAAAA,IAAAA,yBAAAA,8CAAAA,AAAAA,EAA+CJ,QAAQ;QAC5D,SAAS;QACTG;IACF,GAAG,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO;AAG1B,MAAMC,qBAAqB,CAChCN,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAGK,MAAMC,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAGK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACL;QACJ,MAAMT,WAAWe,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BN;QAC7C,OAAO;YACL,aACElB,2BAA2BS,UAAU,uBAAuB;YAC9D,OAAOS,MAAM,KAAK,IAAI;QACxB;IACF,GACC,MAAM,CAAC,CAACO,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAG7C,MAAMC,sBAAsB,CACjCf,QACAgB,4BAAiD,IAAIC,KAAK;IAE1D,IAAIC,aAAa;IACjB,OAAOlB,OAAO,GAAG,CAAC,CAACO,OAAOY;QACxB,MAAMC,gBAAgBpB,MAAM,CAACmB,QAAQ,EAAE;QACvC,MAAME,YAAYrB,MAAM,CAACmB,QAAQ,EAAE;QACnC,MAAMG,gBAAgBtB,OACnB,KAAK,CAAC,GAAGmB,OACT,OAAO,GACP,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAMC,YAAYxB,OACf,KAAK,CAACmB,QAAQ,GACd,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAME,UAAUlB,AAAe,YAAfA,MAAM,IAAI;QAC1B,MAAMmB,qBAAqBD,UAAU,EAAEP,aAAa3B;QACpD,MAAMoC,mBAAmBC,QAAQN,iBAAiBE;QAClD,MAAMK,sBAAsBJ,UACxB;YAACH,eAAe;YAAOE,WAAW;SAAM,CAAC,MAAM,CAC7C,CAAClC,QAA2BsC,QAAQtC,UAEtCC;QACJ,MAAMO,WAAWD,qCACfgB,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BN;QAG9B,OAAO;YACL,QAAQA,MAAM,MAAM;YACpB,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,QAAQA,MAAM,MAAM;YACpB,YAAYA,MAAM,UAAU;YAC5B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClBT;YACA,aAAagC,AAAAA,IAAAA,yBAAAA,mCAAAA,AAAAA,EAAoCvB;YACjD,OAAOA,MAAM,KAAK;YAClB,2BAA2Ba,gBACvBU,AAAAA,IAAAA,yBAAAA,mCAAAA,AAAAA,EAAoCV,iBACpC7B;YACJ,uBAAuB8B,YACnBS,AAAAA,IAAAA,yBAAAA,mCAAAA,AAAAA,EAAoCT,aACpC9B;YACJ,GAAIkC,UACA;gBACE,WAAWlB,MAAM,KAAK,IAAI;gBAC1B,YAAYmB;gBACZ,mBACEN,eAAe,SAAS,WAAWC,WAAW,SAAS;gBACzDM;gBACA,0BAA0BL,gBACtBQ,AAAAA,IAAAA,yBAAAA,mCAAAA,AAAAA,EAAoCR,iBACpC/B;gBACJ,qBACEsC,uBAAuBA,oBAAoB,MAAM,GAAG,IAChDA,sBACAtC;YACR,IACA,CAAC,CAAC;YACN,UAAUgB,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;YAC9B,gBAAgBS,0BAA0B,GAAG,CAACT,MAAM,MAAM;QAC5D;IACF;AACF;AAEO,MAAMwB,sBAAsB,CACjC/B,QACAgC,UAII,CAAC,CAAC;IAEN,MAAMvB,iBAAiBH,mBAAmBN;IAC1C,MAAMiC,cAAczB,kBAAkBC,gBAAgBT,OAAO,MAAM;IAEnE,MAAMkC,WACJzB,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAM0B,oBAAoB1B,eAAe,WAAW,CACjD,GAAG,CAAC,CAACF,QAAUM,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BN,QAAQ,oBACnD,MAAM,CAAC,CAAC6B,OAAyBR,QAAQQ,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAMC,oBAAoB1B,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAM6B,OAAO7B,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAAC8B,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBZ,QAAQY,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkB1B,oBACtBf,QACAgC,QAAQ,yBAAyB;IAGnC,OAAO;QACL,UAAUA,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAK;QACAH;QACAE;QACA,QAAQI;IACV;AACF;AAEO,SAASC,iCACdC,KAA8B;IAE9B5C,eAAe4C,MAAM,MAAM;IAE3B,MAAMxC,iBACJwC,MAAM,cAAc,IAAIC,yBAAAA,kDAAkDA;IAC5E,MAAMC,mBAAmBzC,AAAAA,IAAAA,yBAAAA,8CAAAA,AAAAA,EACvBuC,MAAM,MAAM,EACZ;QACE,SAAS;QACTxC;IACF;IAEF,MAAMa,4BAA4B,IAAIC,IACpC4B,iBAAiB,GAAG,CAAC,CAACxC,QAAU;YAACA,MAAM,WAAW;YAAEA,MAAM,YAAY;SAAC;IAGzE,OAAO;QACL,SAAS0B,oBAAoBY,MAAM,MAAM,EAAE;YACzC,UAAUA,MAAM,QAAQ;YACxB3B;QACF;QACA6B;IACF;AACF;AAEO,MAAMC,uBAAuB,CAClCC,YACAC,cAAwB,EAAE,EAC1BC,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMH;QACR;KACD;IAED,IAAIE,sBAAsBD,YAAY,MAAM,GAAG,GAAG;QAChDE,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAF,YAAY,OAAO,CAAC,CAACG;YACnBD,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKC;gBACP;YACF;QACF;IACF;IAEA,OAAOD;AACT"}

package/dist/lib/ai-model/prompt/recorder-metadata-generator.js

@@ -28,6 +28,7 @@ __webpack_require__.d(__webpack_exports__, {
 });
 const recorder_namespaceObject = require("@midscene/shared/recorder");
 const index_js_namespaceObject = require("../service-caller/index.js");
+const external_recorder_generation_common_js_namespaceObject = require("./recorder-generation-common.js");
 function summarizeRecorderEvents(input) {
     const events = input.events;
     const navigationEvents = events.filter((event)=>'navigation' === event.type);
@@ -56,10 +57,7 @@ function summarizeRecorderEvents(input) {
                 title: event.title,
                 value: event.value,
                 description: (0, recorder_namespaceObject.getMidsceneRecorderEventDescription)(event),
-                elementDescription: event.elementDescription,
-                replayInstruction: event.replayInstruction,
-                actionSummary: event.actionSummary,
-                semanticConfidence: event.semanticConfidence
+                semantic: (0, external_recorder_generation_common_js_namespaceObject.compactRecorderSemanticForGeneration)((0, recorder_namespaceObject.getMidsceneRecorderSemantic)(event))
             }))
     };
 }

package/dist/lib/ai-model/prompt/recorder-metadata-generator.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/recorder-metadata-generator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/recorder-metadata-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type MidsceneRecorderEvent,\n  type MidsceneRecorderTarget,\n  getMidsceneRecorderEventDescription,\n  getMidsceneRecorderScreenshotsForLLM,\n} from '@midscene/shared/recorder';\nimport { callAIWithObjectResponse } from '../service-caller/index';\n\nexport interface RecorderMetadataGenerationInput {\n  target: MidsceneRecorderTarget;\n  events: MidsceneRecorderEvent[];\n  fallbackName?: string;\n  maxScreenshots?: number;\n}\n\nexport interface RecorderGeneratedMetadata {\n  title?: string;\n  description?: string;\n}\n\nfunction summarizeRecorderEvents(input: RecorderMetadataGenerationInput) {\n  const events = input.events;\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const clickEvents = events.filter((event) => event.type === 'click');\n  const inputEvents = events.filter((event) => event.type === 'input');\n  const scrollEvents = events.filter((event) => event.type === 'scroll');\n  const urls = navigationEvents\n    .map((event) => event.url)\n    .filter((url): url is string => Boolean(url));\n  const titles = navigationEvents\n    .map((event) => event.title)\n    .filter((title): title is string => Boolean(title));\n\n  return {\n    platform: input.target.platformId,\n    target: input.target,\n    fallbackName: input.fallbackName,\n    pageCount: navigationEvents.length,\n    pageTitles: titles.slice(0, 5),\n    urls: urls.slice(0, 5),\n    clickCount: clickEvents.length,\n    inputCount: inputEvents.length,\n    scrollCount: scrollEvents.length,\n    totalActions: events.length,\n    firstUrl: urls[0] || input.target.values.url || '',\n    lastUrl: urls[urls.length - 1] || '',\n    events: events.slice(0, 20).map((event) => ({\n      type: event.type,\n      actionType: event.actionType,\n      url: event.url,\n      title: event.title,\n      value: event.value,\n      description: getMidsceneRecorderEventDescription(event),\n      elementDescription: event.elementDescription,\n      replayInstruction: event.replayInstruction,\n      actionSummary: event.actionSummary,\n      semanticConfidence: event.semanticConfidence,\n    })),\n  };\n}\n\nfunction normalizeMetadataValue(value: unknown) {\n  return typeof value === 'string' ? value.trim() : '';\n}\n\nexport async function generateRecorderSessionMetadata(\n  input: RecorderMetadataGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<RecorderGeneratedMetadata> {\n  if (!input?.events?.length) {\n    throw new Error('generateRecorderSessionMetadata: events are required.');\n  }\n  if (!modelConfig?.modelName) {\n    throw new Error(\n      'generateRecorderSessionMetadata: modelConfig.modelName is required.',\n    );\n  }\n\n  const summary = summarizeRecorderEvents(input);\n  const screenshots = getMidsceneRecorderScreenshotsForLLM(\n    input.events,\n    input.maxScreenshots ?? 1,\n  );\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: `Generate a concise title (5-7 words) and brief description (1-2 sentences) for a Studio recording of user actions.\n\nThe recording can target Web, Android, iOS, HarmonyOS, or Computer. Do not assume it is a browser session unless the platform is web.\nDescribe what the user did or accomplished. The description should use the user as the subject, preferably starting with \"The user ...\". Do not start the description with \"The session ...\".\nThe title should be action-oriented and highlight the main task accomplished.\n\nSummary:\n${JSON.stringify(summary, null, 2)}\n\nRespond with a JSON object containing exactly \"title\" and \"description\".`,\n    },\n  ];\n\n  for (const screenshot of screenshots) {\n    messageContent.push({\n      type: 'image_url',\n      image_url: { url: screenshot },\n    });\n  }\n\n  const response = await callAIWithObjectResponse<{\n    title?: string;\n    description?: string;\n  }>(\n    [\n      {\n        role: 'system',\n        content:\n          'You generate clear, task-oriented titles and descriptions for recorded automation sessions.',\n      },\n      {\n        role: 'user',\n        content: messageContent,\n      },\n    ],\n    modelConfig,\n  );\n\n  return {\n    title: normalizeMetadataValue(response.content.title),\n    description: normalizeMetadataValue(response.content.description),\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","summarizeRecorderEvents","input","events","navigationEvents","event","clickEvents","inputEvents","scrollEvents","urls","url","Boolean","titles","title","getMidsceneRecorderEventDescription","normalizeMetadataValue","value","generateRecorderSessionMetadata","modelConfig","Error","summary","screenshots","getMidsceneRecorderScreenshotsForLLM","messageContent","JSON","screenshot","response","callAIWithObjectResponse"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACeA,SAASI,wBAAwBC,KAAsC;IACrE,MAAMC,SAASD,MAAM,MAAM;IAC3B,MAAME,mBAAmBD,OAAO,MAAM,CACpC,CAACE,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,cAAcH,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAME,cAAcJ,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAMG,eAAeL,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACxD,MAAMI,OAAOL,iBACV,GAAG,CAAC,CAACC,QAAUA,MAAM,GAAG,EACxB,MAAM,CAAC,CAACK,MAAuBC,QAAQD;IAC1C,MAAME,SAASR,iBACZ,GAAG,CAAC,CAACC,QAAUA,MAAM,KAAK,EAC1B,MAAM,CAAC,CAACQ,QAA2BF,QAAQE;IAE9C,OAAO;QACL,UAAUX,MAAM,MAAM,CAAC,UAAU;QACjC,QAAQA,MAAM,MAAM;QACpB,cAAcA,MAAM,YAAY;QAChC,WAAWE,iBAAiB,MAAM;QAClC,YAAYQ,OAAO,KAAK,CAAC,GAAG;QAC5B,MAAMH,KAAK,KAAK,CAAC,GAAG;QACpB,YAAYH,YAAY,MAAM;QAC9B,YAAYC,YAAY,MAAM;QAC9B,aAAaC,aAAa,MAAM;QAChC,cAAcL,OAAO,MAAM;QAC3B,UAAUM,IAAI,CAAC,EAAE,IAAIP,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI;QAChD,SAASO,IAAI,CAACA,KAAK,MAAM,GAAG,EAAE,IAAI;QAClC,QAAQN,OAAO,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAACE,QAAW;gBAC1C,MAAMA,MAAM,IAAI;gBAChB,YAAYA,MAAM,UAAU;gBAC5B,KAAKA,MAAM,GAAG;gBACd,OAAOA,MAAM,KAAK;gBAClB,OAAOA,MAAM,KAAK;gBAClB,aAAaS,AAAAA,IAAAA,yBAAAA,mCAAAA,AAAAA,EAAoCT;gBACjD,oBAAoBA,MAAM,kBAAkB;gBAC5C,mBAAmBA,MAAM,iBAAiB;gBAC1C,eAAeA,MAAM,aAAa;gBAClC,oBAAoBA,MAAM,kBAAkB;YAC9C;IACF;AACF;AAEA,SAASU,uBAAuBC,KAAc;IAC5C,OAAO,AAAiB,YAAjB,OAAOA,QAAqBA,MAAM,IAAI,KAAK;AACpD;AAEO,eAAeC,gCACpBf,KAAsC,EACtCgB,WAAyB;IAEzB,IAAI,CAAChB,OAAO,QAAQ,QAClB,MAAM,IAAIiB,MAAM;IAElB,IAAI,CAACD,aAAa,WAChB,MAAM,IAAIC,MACR;IAIJ,MAAMC,UAAUnB,wBAAwBC;IACxC,MAAMmB,cAAcC,AAAAA,IAAAA,yBAAAA,oCAAAA,AAAAA,EAClBpB,MAAM,MAAM,EACZA,MAAM,cAAc,IAAI;IAE1B,MAAMqB,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAM,CAAC;;;;;;;AAOb,EAAEC,KAAK,SAAS,CAACJ,SAAS,MAAM,GAAG;;wEAEqC,CAAC;QACrE;KACD;IAED,KAAK,MAAMK,cAAcJ,YACvBE,eAAe,IAAI,CAAC;QAClB,MAAM;QACN,WAAW;YAAE,KAAKE;QAAW;IAC/B;IAGF,MAAMC,WAAW,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAIrB;QACE;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACN,SAASJ;QACX;KACD,EACDL;IAGF,OAAO;QACL,OAAOH,uBAAuBW,SAAS,OAAO,CAAC,KAAK;QACpD,aAAaX,uBAAuBW,SAAS,OAAO,CAAC,WAAW;IAClE;AACF"}
+{"version":3,"file":"ai-model/prompt/recorder-metadata-generator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/recorder-metadata-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type MidsceneRecorderEvent,\n  type MidsceneRecorderTarget,\n  getMidsceneRecorderEventDescription,\n  getMidsceneRecorderScreenshotsForLLM,\n  getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\nimport { callAIWithObjectResponse } from '../service-caller/index';\nimport { compactRecorderSemanticForGeneration } from './recorder-generation-common';\n\nexport interface RecorderMetadataGenerationInput {\n  target: MidsceneRecorderTarget;\n  events: MidsceneRecorderEvent[];\n  fallbackName?: string;\n  maxScreenshots?: number;\n}\n\nexport interface RecorderGeneratedMetadata {\n  title?: string;\n  description?: string;\n}\n\nfunction summarizeRecorderEvents(input: RecorderMetadataGenerationInput) {\n  const events = input.events;\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const clickEvents = events.filter((event) => event.type === 'click');\n  const inputEvents = events.filter((event) => event.type === 'input');\n  const scrollEvents = events.filter((event) => event.type === 'scroll');\n  const urls = navigationEvents\n    .map((event) => event.url)\n    .filter((url): url is string => Boolean(url));\n  const titles = navigationEvents\n    .map((event) => event.title)\n    .filter((title): title is string => Boolean(title));\n\n  return {\n    platform: input.target.platformId,\n    target: input.target,\n    fallbackName: input.fallbackName,\n    pageCount: navigationEvents.length,\n    pageTitles: titles.slice(0, 5),\n    urls: urls.slice(0, 5),\n    clickCount: clickEvents.length,\n    inputCount: inputEvents.length,\n    scrollCount: scrollEvents.length,\n    totalActions: events.length,\n    firstUrl: urls[0] || input.target.values.url || '',\n    lastUrl: urls[urls.length - 1] || '',\n    events: events.slice(0, 20).map((event) => ({\n      type: event.type,\n      actionType: event.actionType,\n      url: event.url,\n      title: event.title,\n      value: event.value,\n      description: getMidsceneRecorderEventDescription(event),\n      semantic: compactRecorderSemanticForGeneration(\n        getMidsceneRecorderSemantic(event),\n      ),\n    })),\n  };\n}\n\nfunction normalizeMetadataValue(value: unknown) {\n  return typeof value === 'string' ? value.trim() : '';\n}\n\nexport async function generateRecorderSessionMetadata(\n  input: RecorderMetadataGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<RecorderGeneratedMetadata> {\n  if (!input?.events?.length) {\n    throw new Error('generateRecorderSessionMetadata: events are required.');\n  }\n  if (!modelConfig?.modelName) {\n    throw new Error(\n      'generateRecorderSessionMetadata: modelConfig.modelName is required.',\n    );\n  }\n\n  const summary = summarizeRecorderEvents(input);\n  const screenshots = getMidsceneRecorderScreenshotsForLLM(\n    input.events,\n    input.maxScreenshots ?? 1,\n  );\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: `Generate a concise title (5-7 words) and brief description (1-2 sentences) for a Studio recording of user actions.\n\nThe recording can target Web, Android, iOS, HarmonyOS, or Computer. Do not assume it is a browser session unless the platform is web.\nDescribe what the user did or accomplished. The description should use the user as the subject, preferably starting with \"The user ...\". Do not start the description with \"The session ...\".\nThe title should be action-oriented and highlight the main task accomplished.\n\nSummary:\n${JSON.stringify(summary, null, 2)}\n\nRespond with a JSON object containing exactly \"title\" and \"description\".`,\n    },\n  ];\n\n  for (const screenshot of screenshots) {\n    messageContent.push({\n      type: 'image_url',\n      image_url: { url: screenshot },\n    });\n  }\n\n  const response = await callAIWithObjectResponse<{\n    title?: string;\n    description?: string;\n  }>(\n    [\n      {\n        role: 'system',\n        content:\n          'You generate clear, task-oriented titles and descriptions for recorded automation sessions.',\n      },\n      {\n        role: 'user',\n        content: messageContent,\n      },\n    ],\n    modelConfig,\n  );\n\n  return {\n    title: normalizeMetadataValue(response.content.title),\n    description: normalizeMetadataValue(response.content.description),\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","summarizeRecorderEvents","input","events","navigationEvents","event","clickEvents","inputEvents","scrollEvents","urls","url","Boolean","titles","title","getMidsceneRecorderEventDescription","compactRecorderSemanticForGeneration","getMidsceneRecorderSemantic","normalizeMetadataValue","value","generateRecorderSessionMetadata","modelConfig","Error","summary","screenshots","getMidsceneRecorderScreenshotsForLLM","messageContent","JSON","screenshot","response","callAIWithObjectResponse"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACiBA,SAASI,wBAAwBC,KAAsC;IACrE,MAAMC,SAASD,MAAM,MAAM;IAC3B,MAAME,mBAAmBD,OAAO,MAAM,CACpC,CAACE,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,cAAcH,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAME,cAAcJ,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAMG,eAAeL,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACxD,MAAMI,OAAOL,iBACV,GAAG,CAAC,CAACC,QAAUA,MAAM,GAAG,EACxB,MAAM,CAAC,CAACK,MAAuBC,QAAQD;IAC1C,MAAME,SAASR,iBACZ,GAAG,CAAC,CAACC,QAAUA,MAAM,KAAK,EAC1B,MAAM,CAAC,CAACQ,QAA2BF,QAAQE;IAE9C,OAAO;QACL,UAAUX,MAAM,MAAM,CAAC,UAAU;QACjC,QAAQA,MAAM,MAAM;QACpB,cAAcA,MAAM,YAAY;QAChC,WAAWE,iBAAiB,MAAM;QAClC,YAAYQ,OAAO,KAAK,CAAC,GAAG;QAC5B,MAAMH,KAAK,KAAK,CAAC,GAAG;QACpB,YAAYH,YAAY,MAAM;QAC9B,YAAYC,YAAY,MAAM;QAC9B,aAAaC,aAAa,MAAM;QAChC,cAAcL,OAAO,MAAM;QAC3B,UAAUM,IAAI,CAAC,EAAE,IAAIP,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI;QAChD,SAASO,IAAI,CAACA,KAAK,MAAM,GAAG,EAAE,IAAI;QAClC,QAAQN,OAAO,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAACE,QAAW;gBAC1C,MAAMA,MAAM,IAAI;gBAChB,YAAYA,MAAM,UAAU;gBAC5B,KAAKA,MAAM,GAAG;gBACd,OAAOA,MAAM,KAAK;gBAClB,OAAOA,MAAM,KAAK;gBAClB,aAAaS,AAAAA,IAAAA,yBAAAA,mCAAAA,AAAAA,EAAoCT;gBACjD,UAAUU,AAAAA,IAAAA,uDAAAA,oCAAAA,AAAAA,EACRC,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BX;YAEhC;IACF;AACF;AAEA,SAASY,uBAAuBC,KAAc;IAC5C,OAAO,AAAiB,YAAjB,OAAOA,QAAqBA,MAAM,IAAI,KAAK;AACpD;AAEO,eAAeC,gCACpBjB,KAAsC,EACtCkB,WAAyB;IAEzB,IAAI,CAAClB,OAAO,QAAQ,QAClB,MAAM,IAAImB,MAAM;IAElB,IAAI,CAACD,aAAa,WAChB,MAAM,IAAIC,MACR;IAIJ,MAAMC,UAAUrB,wBAAwBC;IACxC,MAAMqB,cAAcC,AAAAA,IAAAA,yBAAAA,oCAAAA,AAAAA,EAClBtB,MAAM,MAAM,EACZA,MAAM,cAAc,IAAI;IAE1B,MAAMuB,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAM,CAAC;;;;;;;AAOb,EAAEC,KAAK,SAAS,CAACJ,SAAS,MAAM,GAAG;;wEAEqC,CAAC;QACrE;KACD;IAED,KAAK,MAAMK,cAAcJ,YACvBE,eAAe,IAAI,CAAC;QAClB,MAAM;QACN,WAAW;YAAE,KAAKE;QAAW;IAC/B;IAGF,MAAMC,WAAW,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAIrB;QACE;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACN,SAASJ;QACX;KACD,EACDL;IAGF,OAAO;QACL,OAAOH,uBAAuBW,SAAS,OAAO,CAAC,KAAK;QACpD,aAAaX,uBAAuBW,SAAS,OAAO,CAAC,WAAW;IAClE;AACF"}