@midscene/core 1.13.1 → 1.13.2-beta-20260923031905.0

package/dist/es/ai-model/workflows/recorder-generation/markdown.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/recorder-generation/markdown.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/markdown.ts"],"sourcesContent":["import type { IModelConfig } from '@midscene/shared/env';\nimport {\n  imageInfoOfBase64,\n  parseBase64,\n  resizeBase64ImageToJpeg,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport {\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  getMidsceneRecorderEventDescription,\n  stringifyMidsceneRecorderTargetBlock,\n} from '@midscene/shared/recorder';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { getModelRuntime } from '../../models';\nimport { callAIWithStringResponse } from '../../service-caller';\nimport {\n  type RecorderGenerationInput,\n  prepareRecorderGenerationContext,\n  validateEvents,\n} from './common';\n\nexport type RecorderMarkdownGenerationInput = RecorderGenerationInput;\n\nconst MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET = 600_000;\nconst MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE = 768;\nconst debugMarkdownReplay = getDebug('ai:recorder-markdown', {\n  console: true,\n});\n\nfunction limitScreenshotAssetsForMarkdownReplay(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  let usedPayload = 0;\n  return screenshotAssets.filter((asset) => {\n    const payloadSize = asset.dataUrl.length;\n    if (\n      payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET ||\n      usedPayload + payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET\n    ) {\n      return false;\n    }\n    usedPayload += payloadSize;\n    return true;\n  });\n}\n\nasync function compressScreenshotAssetForMarkdownReplay(\n  asset: MidsceneRecorderMarkdownScreenshotAsset,\n): Promise<MidsceneRecorderMarkdownScreenshotAsset> {\n  const { width, height } = await imageInfoOfBase64(asset.dataUrl);\n  const longestEdge = Math.max(width, height);\n  if (longestEdge <= MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE) {\n    return asset;\n  }\n\n  const scale = MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE / longestEdge;\n  const dataUrl = await resizeBase64ImageToJpeg(asset.dataUrl, {\n    sourceSize: { width, height },\n    targetSize: {\n      width: Math.max(1, Math.round(width * scale)),\n      height: Math.max(1, Math.round(height * scale)),\n    },\n  });\n  const { body, mimeType } = parseBase64(dataUrl);\n  return {\n    ...asset,\n    dataUrl,\n    base64Data: body,\n    mimeType,\n  };\n}\n\nasync function prepareScreenshotAssetsForMarkdownReplay(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const compressedAssets: MidsceneRecorderMarkdownScreenshotAsset[] = [];\n  for (const asset of screenshotAssets) {\n    try {\n      compressedAssets.push(\n        await compressScreenshotAssetForMarkdownReplay(asset),\n      );\n    } catch (error) {\n      debugMarkdownReplay(\n        'failed to compress screenshot asset; keeping original %o',\n        {\n          eventHashId: asset.eventHashId,\n          eventIndex: asset.eventIndex,\n          error,\n        },\n      );\n      compressedAssets.push(asset);\n    }\n  }\n  return limitScreenshotAssetsForMarkdownReplay(compressedAssets);\n}\n\nfunction summarizeScreenshotAssets(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const payloadSizes = screenshotAssets.map((asset) => asset.dataUrl.length);\n  return {\n    count: screenshotAssets.length,\n    totalPayloadChars: payloadSizes.reduce((sum, size) => sum + size, 0),\n    maxPayloadChars: payloadSizes.length ? Math.max(...payloadSizes) : 0,\n  };\n}\n\nfunction getPromptShape(prompt: ChatCompletionMessageParam[]) {\n  let textChars = 0;\n  let imageCount = 0;\n  for (const message of prompt) {\n    const content = message.content;\n    if (typeof content === 'string') {\n      textChars += content.length;\n      continue;\n    }\n    if (!Array.isArray(content)) {\n      continue;\n    }\n    for (const part of content) {\n      if (typeof part === 'object' && part && 'type' in part) {\n        if (part.type === 'text' && 'text' in part) {\n          textChars += String(part.text).length;\n        }\n        if (part.type === 'image_url') {\n          imageCount += 1;\n        }\n      }\n    }\n  }\n  return { textChars, imageCount };\n}\n\nfunction removeOmittedScreenshotPaths(\n  summary: ReturnType<typeof prepareRecorderGenerationContext>['summary'],\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const includedScreenshotPaths = new Set(\n    screenshotAssets.map((asset) => asset.relativePath),\n  );\n  return {\n    ...summary,\n    events: summary.events.map((event) =>\n      event.screenshotPath && !includedScreenshotPaths.has(event.screenshotPath)\n        ? { ...event, screenshotPath: undefined }\n        : event,\n    ),\n  };\n}\n\nfunction getMarkdownLanguageInstruction(language?: string) {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable Markdown instructions in ${normalizedLanguage}.\n- Keep file paths, URLs, platform ids, API names, and quoted UI text unchanged.`;\n}\n\nfunction normalizeGeneratedMarkdown(content: string) {\n  const trimmed = content.trim();\n  const fencedMatch = trimmed.match(\n    /^```(?:md|markdown)?\\s*([\\s\\S]*?)\\s*```$/i,\n  );\n  return `${(fencedMatch?.[1] ?? trimmed).trim()}\\n`;\n}\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\nexport function createRecorderMarkdownReplayPrompt(\n  input: RecorderMarkdownGenerationInput,\n): ChatCompletionMessageParam[] {\n  validateEvents(input.events);\n\n  const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } =\n    prepareRecorderGenerationContext(input);\n  const screenshotAssets =\n    limitScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);\n  const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);\n  return createRecorderMarkdownReplayPromptFromContext(\n    input,\n    summary,\n    screenshotAssets,\n  );\n}\n\nasync function createRecorderMarkdownReplayPromptForGeneration(\n  input: RecorderMarkdownGenerationInput,\n): Promise<ChatCompletionMessageParam[]> {\n  validateEvents(input.events);\n\n  const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } =\n    prepareRecorderGenerationContext(input);\n  const screenshotAssets =\n    await prepareScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);\n  const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);\n  const prompt = createRecorderMarkdownReplayPromptFromContext(\n    input,\n    summary,\n    screenshotAssets,\n  );\n  debugMarkdownReplay('markdown replay prompt shape %o', {\n    eventCount: input.events.length,\n    maxScreenshots: input.maxScreenshots,\n    rawScreenshots: summarizeScreenshotAssets(rawScreenshotAssets),\n    includedScreenshots: summarizeScreenshotAssets(screenshotAssets),\n    prompt: getPromptShape(prompt),\n  });\n  return prompt;\n}\n\nfunction createRecorderMarkdownReplayPromptFromContext(\n  input: RecorderMarkdownGenerationInput,\n  summary: ReturnType<typeof prepareRecorderGenerationContext>['summary'],\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n): ChatCompletionMessageParam[] {\n  const screenshotIndexByEventHash = new Map(\n    screenshotAssets.map((asset, index) => [\n      asset.eventHashId,\n      `screenshot-${index + 1}`,\n    ]),\n  );\n  const events = summary.events.map((event) => {\n    const screenshotRef = screenshotIndexByEventHash.get(event.hashId);\n    const { screenshotPath, ...eventWithoutScreenshotPath } = event;\n    return screenshotRef\n      ? { ...eventWithoutScreenshotPath, screenshotRef }\n      : eventWithoutScreenshotPath;\n  });\n  const promptPayload = {\n    testName: input.testName || summary.testName,\n    target: {\n      platformId: input.target.platformId,\n      label: input.target.label,\n      values: input.target.values,\n    },\n    startUrl: summary.startUrl,\n    events,\n    screenshots: screenshotAssets.map((asset, index) => ({\n      screenshotRef: `screenshot-${index + 1}`,\n      eventIndex: asset.eventIndex,\n      eventHashId: asset.eventHashId,\n      eventType: asset.eventType,\n      description: getMidsceneRecorderEventDescription(\n        input.events[asset.eventIndex],\n      ),\n    })),\n  };\n  const promptText = `Generate a Markdown replay script for Midscene Agent. It will be executed with:\nawait agent.aiAct(markdownReplayPrompt)\n\nUse only the recorder data and screenshots below.\n\nTarget block:\n${stringifyMidsceneRecorderTargetBlock(input.target)}\n\nReplay goal:\n- Reproduce the recorded user workflow exactly.\n- Preserve event order.\n- Preserve the user's original intent.\n- Do not invent alternative navigation paths.\n- Do not skip, merge, reorder, or add extra user actions.\n- Prefer recorded UI text, element descriptions, URLs, input values, and scroll direction.\n- For input events, enter event.typedText/event.value exactly; do not infer or correct the text from screenshots.\n- Prefer event.semantic.replayInstruction and event.semantic.elementDescription when event.semantic.source is \"aiDescribe\" or \"recorderAI\" and event.semantic.status is \"ready\".\n- For scroll events, preserve the recorded scroll region from event.semantic.elementDescription/replayInstruction. If the scroll happened in a specific panel, list, table, dialog body, menu, navigation area, or content pane, keep that region in the Markdown step instead of generalizing it to the whole page.\n- If event.semantic.source is \"heuristic\" or event.semantic.status is \"pending\"/\"failed\", use the screenshot/context to write the best visual instruction.\n- Coordinates are only fallback hints. Do not make coordinates the primary instruction when text or screenshots are available.\n- For a click/tap that only focuses a field before an input event, describe the target as the field/control itself. Do not target a placeholder character, typed character, caret, or inner text fragment inside the field.\n- If a target cannot be found, stop and report the missing step. Do not click similar-looking elements.\n- Screenshots are only generation-time visual evidence for you. The generated Markdown will be passed directly to agent.aiAct(markdownReplayPrompt), which accepts text only and cannot receive attached images.\n- Convert any useful screenshot evidence into textual replay instructions. Do not include screenshots, image syntax, image paths, or reference-image names in the generated Markdown.\n- Never write Markdown image syntax such as ![step context](...), reference-style images, HTML <img> tags, ./screenshots/... paths, or screenshot-* names in the output.\n\nRequired structure:\n# ${input.testName || summary.testName}\n\n## Goal\nReproduce the recorded user workflow exactly.\n\n## Target\n- Platform: ${input.target.platformId}\n- Start target: ${summary.startUrl || input.target.label || input.target.deviceId || 'Recorded target'}\n\n## Steps\n1. ...\n\nRecorder data:\n${JSON.stringify(promptPayload, null, 2)}${getMarkdownLanguageInstruction(input.language)}\n\nImportant: Return ONLY raw Markdown. Do NOT wrap the response in markdown code blocks.`;\n\n  const content: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  for (const asset of screenshotAssets) {\n    const screenshotRef = screenshotIndexByEventHash.get(asset.eventHashId);\n    content.push({\n      type: 'text',\n      text: `${screenshotRef} for event #${asset.eventIndex + 1}`,\n    });\n    content.push({\n      type: 'image_url',\n      image_url: {\n        url: asset.dataUrl,\n      },\n    });\n  }\n\n  return [\n    {\n      role: 'system',\n      content:\n        'You generate precise Markdown replay scripts for Midscene agent.aiAct. The final output is plain text that will be passed directly to agent.aiAct, so it must be deterministic, ordered, safe for AI execution, and must not contain image references, screenshot paths, or screenshot labels.',\n    },\n    {\n      role: 'user',\n      content,\n    },\n  ];\n}\n\nexport async function generateRecorderMarkdownReplay(\n  input: RecorderMarkdownGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> {\n  try {\n    const prompt = await createRecorderMarkdownReplayPromptForGeneration(input);\n    const response = await callAIWithStringResponse(\n      prompt,\n      createModelRuntime(modelConfig),\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return normalizeGeneratedMarkdown(response.content);\n    }\n\n    throw new Error('Failed to generate recorder Markdown replay');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder Markdown replay: ${error}`);\n  }\n}\n\nexport async function convertRecordLogIntoMarkdown(\n  log: RecorderMarkdownGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> {\n  return generateRecorderMarkdownReplay(log, modelConfig);\n}\n"],"names":["MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET","MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE","debugMarkdownReplay","getDebug","limitScreenshotAssetsForMarkdownReplay","screenshotAssets","usedPayload","asset","payloadSize","compressScreenshotAssetForMarkdownReplay","width","height","imageInfoOfBase64","longestEdge","Math","scale","dataUrl","resizeBase64ImageToJpeg","body","mimeType","parseBase64","prepareScreenshotAssetsForMarkdownReplay","compressedAssets","error","summarizeScreenshotAssets","payloadSizes","sum","size","getPromptShape","prompt","textChars","imageCount","message","content","Array","part","String","removeOmittedScreenshotPaths","summary","includedScreenshotPaths","Set","event","undefined","getMarkdownLanguageInstruction","language","normalizedLanguage","normalizeGeneratedMarkdown","trimmed","fencedMatch","createModelRuntime","modelConfig","getModelRuntime","createRecorderMarkdownReplayPrompt","input","validateEvents","rawSummary","rawScreenshotAssets","prepareRecorderGenerationContext","createRecorderMarkdownReplayPromptFromContext","createRecorderMarkdownReplayPromptForGeneration","screenshotIndexByEventHash","Map","index","events","screenshotRef","screenshotPath","eventWithoutScreenshotPath","promptPayload","getMidsceneRecorderEventDescription","promptText","stringifyMidsceneRecorderTargetBlock","JSON","generateRecorderMarkdownReplay","response","callAIWithStringResponse","Error","convertRecordLogIntoMarkdown","log"],"mappings":";;;;;;AAuBA,MAAMA,4CAA4C;AAClD,MAAMC,sCAAsC;AAC5C,MAAMC,sBAAsBC,SAAS,wBAAwB;IAC3D,SAAS;AACX;AAEA,SAASC,uCACPC,gBAA2D;IAE3D,IAAIC,cAAc;IAClB,OAAOD,iBAAiB,MAAM,CAAC,CAACE;QAC9B,MAAMC,cAAcD,MAAM,OAAO,CAAC,MAAM;QACxC,IACEC,cAAcR,6CACdM,cAAcE,cAAcR,2CAE5B,OAAO;QAETM,eAAeE;QACf,OAAO;IACT;AACF;AAEA,eAAeC,yCACbF,KAA8C;IAE9C,MAAM,EAAEG,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMC,kBAAkBL,MAAM,OAAO;IAC/D,MAAMM,cAAcC,KAAK,GAAG,CAACJ,OAAOC;IACpC,IAAIE,eAAeZ,qCACjB,OAAOM;IAGT,MAAMQ,QAAQd,sCAAsCY;IACpD,MAAMG,UAAU,MAAMC,wBAAwBV,MAAM,OAAO,EAAE;QAC3D,YAAY;YAAEG;YAAOC;QAAO;QAC5B,YAAY;YACV,OAAOG,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACJ,QAAQK;YACtC,QAAQD,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACH,SAASI;QAC1C;IACF;IACA,MAAM,EAAEG,IAAI,EAAEC,QAAQ,EAAE,GAAGC,YAAYJ;IACvC,OAAO;QACL,GAAGT,KAAK;QACRS;QACA,YAAYE;QACZC;IACF;AACF;AAEA,eAAeE,yCACbhB,gBAA2D;IAE3D,MAAMiB,mBAA8D,EAAE;IACtE,KAAK,MAAMf,SAASF,iBAClB,IAAI;QACFiB,iBAAiB,IAAI,CACnB,MAAMb,yCAAyCF;IAEnD,EAAE,OAAOgB,OAAO;QACdrB,oBACE,4DACA;YACE,aAAaK,MAAM,WAAW;YAC9B,YAAYA,MAAM,UAAU;YAC5BgB;QACF;QAEFD,iBAAiB,IAAI,CAACf;IACxB;IAEF,OAAOH,uCAAuCkB;AAChD;AAEA,SAASE,0BACPnB,gBAA2D;IAE3D,MAAMoB,eAAepB,iBAAiB,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO,CAAC,MAAM;IACzE,OAAO;QACL,OAAOF,iBAAiB,MAAM;QAC9B,mBAAmBoB,aAAa,MAAM,CAAC,CAACC,KAAKC,OAASD,MAAMC,MAAM;QAClE,iBAAiBF,aAAa,MAAM,GAAGX,KAAK,GAAG,IAAIW,gBAAgB;IACrE;AACF;AAEA,SAASG,eAAeC,MAAoC;IAC1D,IAAIC,YAAY;IAChB,IAAIC,aAAa;IACjB,KAAK,MAAMC,WAAWH,OAAQ;QAC5B,MAAMI,UAAUD,QAAQ,OAAO;QAC/B,IAAI,AAAmB,YAAnB,OAAOC,SAAsB;YAC/BH,aAAaG,QAAQ,MAAM;YAC3B;QACF;QACA,IAAKC,MAAM,OAAO,CAACD,UAGnB;YAAA,KAAK,MAAME,QAAQF,QACjB,IAAI,AAAgB,YAAhB,OAAOE,QAAqBA,QAAQ,UAAUA,MAAM;gBACtD,IAAIA,AAAc,WAAdA,KAAK,IAAI,IAAe,UAAUA,MACpCL,aAAaM,OAAOD,KAAK,IAAI,EAAE,MAAM;gBAEvC,IAAIA,AAAc,gBAAdA,KAAK,IAAI,EACXJ,cAAc;YAElB;QACF;IACF;IACA,OAAO;QAAED;QAAWC;IAAW;AACjC;AAEA,SAASM,6BACPC,OAAuE,EACvEjC,gBAA2D;IAE3D,MAAMkC,0BAA0B,IAAIC,IAClCnC,iBAAiB,GAAG,CAAC,CAACE,QAAUA,MAAM,YAAY;IAEpD,OAAO;QACL,GAAG+B,OAAO;QACV,QAAQA,QAAQ,MAAM,CAAC,GAAG,CAAC,CAACG,QAC1BA,MAAM,cAAc,IAAI,CAACF,wBAAwB,GAAG,CAACE,MAAM,cAAc,IACrE;gBAAE,GAAGA,KAAK;gBAAE,gBAAgBC;YAAU,IACtCD;IAER;AACF;AAEA,SAASE,+BAA+BC,QAAiB;IACvD,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;oDAE0C,EAAEA,mBAAmB;+EACM,CAAC;AAChF;AAEA,SAASC,2BAA2Bb,OAAe;IACjD,MAAMc,UAAUd,QAAQ,IAAI;IAC5B,MAAMe,cAAcD,QAAQ,KAAK,CAC/B;IAEF,OAAO,GAAIC,AAAAA,CAAAA,aAAa,CAAC,EAAE,IAAID,OAAM,EAAG,IAAI,GAAG,EAAE,CAAC;AACpD;AAEA,SAASE,mBAAmBC,WAAyB;IACnD,OAAOC,gBAAgBD;AACzB;AAEO,SAASE,mCACdC,KAAsC;IAEtCC,eAAeD,MAAM,MAAM;IAE3B,MAAM,EAAE,SAASE,UAAU,EAAE,kBAAkBC,mBAAmB,EAAE,GAClEC,iCAAiCJ;IACnC,MAAMhD,mBACJD,uCAAuCoD;IACzC,MAAMlB,UAAUD,6BAA6BkB,YAAYlD;IACzD,OAAOqD,8CACLL,OACAf,SACAjC;AAEJ;AAEA,eAAesD,gDACbN,KAAsC;IAEtCC,eAAeD,MAAM,MAAM;IAE3B,MAAM,EAAE,SAASE,UAAU,EAAE,kBAAkBC,mBAAmB,EAAE,GAClEC,iCAAiCJ;IACnC,MAAMhD,mBACJ,MAAMgB,yCAAyCmC;IACjD,MAAMlB,UAAUD,6BAA6BkB,YAAYlD;IACzD,MAAMwB,SAAS6B,8CACbL,OACAf,SACAjC;IAEFH,oBAAoB,mCAAmC;QACrD,YAAYmD,MAAM,MAAM,CAAC,MAAM;QAC/B,gBAAgBA,MAAM,cAAc;QACpC,gBAAgB7B,0BAA0BgC;QAC1C,qBAAqBhC,0BAA0BnB;QAC/C,QAAQuB,eAAeC;IACzB;IACA,OAAOA;AACT;AAEA,SAAS6B,8CACPL,KAAsC,EACtCf,OAAuE,EACvEjC,gBAA2D;IAE3D,MAAMuD,6BAA6B,IAAIC,IACrCxD,iBAAiB,GAAG,CAAC,CAACE,OAAOuD,QAAU;YACrCvD,MAAM,WAAW;YACjB,CAAC,WAAW,EAAEuD,QAAQ,GAAG;SAC1B;IAEH,MAAMC,SAASzB,QAAQ,MAAM,CAAC,GAAG,CAAC,CAACG;QACjC,MAAMuB,gBAAgBJ,2BAA2B,GAAG,CAACnB,MAAM,MAAM;QACjE,MAAM,EAAEwB,cAAc,EAAE,GAAGC,4BAA4B,GAAGzB;QAC1D,OAAOuB,gBACH;YAAE,GAAGE,0BAA0B;YAAEF;QAAc,IAC/CE;IACN;IACA,MAAMC,gBAAgB;QACpB,UAAUd,MAAM,QAAQ,IAAIf,QAAQ,QAAQ;QAC5C,QAAQ;YACN,YAAYe,MAAM,MAAM,CAAC,UAAU;YACnC,OAAOA,MAAM,MAAM,CAAC,KAAK;YACzB,QAAQA,MAAM,MAAM,CAAC,MAAM;QAC7B;QACA,UAAUf,QAAQ,QAAQ;QAC1ByB;QACA,aAAa1D,iBAAiB,GAAG,CAAC,CAACE,OAAOuD,QAAW;gBACnD,eAAe,CAAC,WAAW,EAAEA,QAAQ,GAAG;gBACxC,YAAYvD,MAAM,UAAU;gBAC5B,aAAaA,MAAM,WAAW;gBAC9B,WAAWA,MAAM,SAAS;gBAC1B,aAAa6D,oCACXf,MAAM,MAAM,CAAC9C,MAAM,UAAU,CAAC;YAElC;IACF;IACA,MAAM8D,aAAa,CAAC;;;;;;AAMtB,EAAEC,qCAAqCjB,MAAM,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;EAqBnD,EAAEA,MAAM,QAAQ,IAAIf,QAAQ,QAAQ,CAAC;;;;;;YAM3B,EAAEe,MAAM,MAAM,CAAC,UAAU,CAAC;gBACtB,EAAEf,QAAQ,QAAQ,IAAIe,MAAM,MAAM,CAAC,KAAK,IAAIA,MAAM,MAAM,CAAC,QAAQ,IAAI,kBAAkB;;;;;;AAMvG,EAAEkB,KAAK,SAAS,CAACJ,eAAe,MAAM,KAAKxB,+BAA+BU,MAAM,QAAQ,EAAE;;sFAEJ,CAAC;IAErF,MAAMpB,UAAiB;QACrB;YACE,MAAM;YACN,MAAMoC;QACR;KACD;IAED,KAAK,MAAM9D,SAASF,iBAAkB;QACpC,MAAM2D,gBAAgBJ,2BAA2B,GAAG,CAACrD,MAAM,WAAW;QACtE0B,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,GAAG+B,cAAc,YAAY,EAAEzD,MAAM,UAAU,GAAG,GAAG;QAC7D;QACA0B,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,WAAW;gBACT,KAAK1B,MAAM,OAAO;YACpB;QACF;IACF;IAEA,OAAO;QACL;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACN0B;QACF;KACD;AACH;AAEO,eAAeuC,+BACpBnB,KAAsC,EACtCH,WAAyB;IAEzB,IAAI;QACF,MAAMrB,SAAS,MAAM8B,gDAAgDN;QACrE,MAAMoB,WAAW,MAAMC,yBACrB7C,QACAoB,mBAAmBC;QAGrB,IAAIuB,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO3B,2BAA2B2B,SAAS,OAAO;QAGpD,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOpD,OAAO;QACd,MAAM,IAAIoD,MAAM,CAAC,6CAA6C,EAAEpD,OAAO;IACzE;AACF;AAEO,eAAeqD,6BACpBC,GAAoC,EACpC3B,WAAyB;IAEzB,OAAOsB,+BAA+BK,KAAK3B;AAC7C"}
+{"version":3,"file":"ai-model/workflows/recorder-generation/markdown.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/markdown.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  imageInfoOfBase64,\n  parseBase64,\n  resizeBase64ImageToJpeg,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport {\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  getMidsceneRecorderEventDescription,\n  stringifyMidsceneRecorderTargetBlock,\n} from '@midscene/shared/recorder';\nimport { getModelRuntime } from '../../models';\nimport { callAIWithStringResponse } from '../../service-caller';\nimport {\n  type RecorderGenerationInput,\n  prepareRecorderGenerationContext,\n  validateEvents,\n} from './common';\n\nexport type RecorderMarkdownGenerationInput = RecorderGenerationInput;\n\nconst MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET = 600_000;\nconst MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE = 768;\nconst debugMarkdownReplay = getDebug('ai:recorder-markdown', {\n  console: true,\n});\n\nfunction limitScreenshotAssetsForMarkdownReplay(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  let usedPayload = 0;\n  return screenshotAssets.filter((asset) => {\n    const payloadSize = asset.dataUrl.length;\n    if (\n      payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET ||\n      usedPayload + payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET\n    ) {\n      return false;\n    }\n    usedPayload += payloadSize;\n    return true;\n  });\n}\n\nasync function compressScreenshotAssetForMarkdownReplay(\n  asset: MidsceneRecorderMarkdownScreenshotAsset,\n): Promise<MidsceneRecorderMarkdownScreenshotAsset> {\n  const { width, height } = await imageInfoOfBase64(asset.dataUrl);\n  const longestEdge = Math.max(width, height);\n  if (longestEdge <= MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE) {\n    return asset;\n  }\n\n  const scale = MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE / longestEdge;\n  const dataUrl = await resizeBase64ImageToJpeg(asset.dataUrl, {\n    sourceSize: { width, height },\n    targetSize: {\n      width: Math.max(1, Math.round(width * scale)),\n      height: Math.max(1, Math.round(height * scale)),\n    },\n  });\n  const { body, mimeType } = parseBase64(dataUrl);\n  return {\n    ...asset,\n    dataUrl,\n    base64Data: body,\n    mimeType,\n  };\n}\n\nasync function prepareScreenshotAssetsForMarkdownReplay(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const compressedAssets: MidsceneRecorderMarkdownScreenshotAsset[] = [];\n  for (const asset of screenshotAssets) {\n    try {\n      compressedAssets.push(\n        await compressScreenshotAssetForMarkdownReplay(asset),\n      );\n    } catch (error) {\n      debugMarkdownReplay(\n        'failed to compress screenshot asset; keeping original %o',\n        {\n          eventHashId: asset.eventHashId,\n          eventIndex: asset.eventIndex,\n          error,\n        },\n      );\n      compressedAssets.push(asset);\n    }\n  }\n  return limitScreenshotAssetsForMarkdownReplay(compressedAssets);\n}\n\nfunction summarizeScreenshotAssets(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const payloadSizes = screenshotAssets.map((asset) => asset.dataUrl.length);\n  return {\n    count: screenshotAssets.length,\n    totalPayloadChars: payloadSizes.reduce((sum, size) => sum + size, 0),\n    maxPayloadChars: payloadSizes.length ? Math.max(...payloadSizes) : 0,\n  };\n}\n\nfunction getPromptShape(prompt: ConversationMessage[]) {\n  let textChars = 0;\n  let imageCount = 0;\n  for (const message of prompt) {\n    const content = message.content;\n    if (typeof content === 'string') {\n      textChars += content.length;\n      continue;\n    }\n    if (!Array.isArray(content)) {\n      continue;\n    }\n    for (const part of content) {\n      if (typeof part === 'object' && part && 'type' in part) {\n        if (part.type === 'text' && 'text' in part) {\n          textChars += String(part.text).length;\n        }\n        if (part.type === 'image') {\n          imageCount += 1;\n        }\n      }\n    }\n  }\n  return { textChars, imageCount };\n}\n\nfunction removeOmittedScreenshotPaths(\n  summary: ReturnType<typeof prepareRecorderGenerationContext>['summary'],\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const includedScreenshotPaths = new Set(\n    screenshotAssets.map((asset) => asset.relativePath),\n  );\n  return {\n    ...summary,\n    events: summary.events.map((event) =>\n      event.screenshotPath && !includedScreenshotPaths.has(event.screenshotPath)\n        ? { ...event, screenshotPath: undefined }\n        : event,\n    ),\n  };\n}\n\nfunction getMarkdownLanguageInstruction(language?: string) {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable Markdown instructions in ${normalizedLanguage}.\n- Keep file paths, URLs, platform ids, API names, and quoted UI text unchanged.`;\n}\n\nfunction normalizeGeneratedMarkdown(content: string) {\n  const trimmed = content.trim();\n  const fencedMatch = trimmed.match(\n    /^```(?:md|markdown)?\\s*([\\s\\S]*?)\\s*```$/i,\n  );\n  return `${(fencedMatch?.[1] ?? trimmed).trim()}\\n`;\n}\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\nexport function createRecorderMarkdownReplayPrompt(\n  input: RecorderMarkdownGenerationInput,\n): ConversationMessage[] {\n  validateEvents(input.events);\n\n  const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } =\n    prepareRecorderGenerationContext(input);\n  const screenshotAssets =\n    limitScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);\n  const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);\n  return createRecorderMarkdownReplayPromptFromContext(\n    input,\n    summary,\n    screenshotAssets,\n  );\n}\n\nasync function createRecorderMarkdownReplayPromptForGeneration(\n  input: RecorderMarkdownGenerationInput,\n): Promise<ConversationMessage[]> {\n  validateEvents(input.events);\n\n  const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } =\n    prepareRecorderGenerationContext(input);\n  const screenshotAssets =\n    await prepareScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);\n  const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);\n  const prompt = createRecorderMarkdownReplayPromptFromContext(\n    input,\n    summary,\n    screenshotAssets,\n  );\n  debugMarkdownReplay('markdown replay prompt shape %o', {\n    eventCount: input.events.length,\n    maxScreenshots: input.maxScreenshots,\n    rawScreenshots: summarizeScreenshotAssets(rawScreenshotAssets),\n    includedScreenshots: summarizeScreenshotAssets(screenshotAssets),\n    prompt: getPromptShape(prompt),\n  });\n  return prompt;\n}\n\nfunction createRecorderMarkdownReplayPromptFromContext(\n  input: RecorderMarkdownGenerationInput,\n  summary: ReturnType<typeof prepareRecorderGenerationContext>['summary'],\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n): ConversationMessage[] {\n  const screenshotIndexByEventHash = new Map(\n    screenshotAssets.map((asset, index) => [\n      asset.eventHashId,\n      `screenshot-${index + 1}`,\n    ]),\n  );\n  const events = summary.events.map((event) => {\n    const screenshotRef = screenshotIndexByEventHash.get(event.hashId);\n    const { screenshotPath, ...eventWithoutScreenshotPath } = event;\n    return screenshotRef\n      ? { ...eventWithoutScreenshotPath, screenshotRef }\n      : eventWithoutScreenshotPath;\n  });\n  const promptPayload = {\n    testName: input.testName || summary.testName,\n    target: {\n      platformId: input.target.platformId,\n      label: input.target.label,\n      values: input.target.values,\n    },\n    startUrl: summary.startUrl,\n    events,\n    screenshots: screenshotAssets.map((asset, index) => ({\n      screenshotRef: `screenshot-${index + 1}`,\n      eventIndex: asset.eventIndex,\n      eventHashId: asset.eventHashId,\n      eventType: asset.eventType,\n      description: getMidsceneRecorderEventDescription(\n        input.events[asset.eventIndex],\n      ),\n    })),\n  };\n  const promptText = `Generate a Markdown replay script for Midscene Agent. It will be executed with:\nawait agent.aiAct(markdownReplayPrompt)\n\nUse only the recorder data and screenshots below.\n\nTarget block:\n${stringifyMidsceneRecorderTargetBlock(input.target)}\n\nReplay goal:\n- Reproduce the recorded user workflow exactly.\n- Preserve event order.\n- Preserve the user's original intent.\n- Do not invent alternative navigation paths.\n- Do not skip, merge, reorder, or add extra user actions.\n- Prefer recorded UI text, element descriptions, URLs, input values, and scroll direction.\n- For input events, enter event.typedText/event.value exactly; do not infer or correct the text from screenshots.\n- Prefer event.semantic.replayInstruction and event.semantic.elementDescription when event.semantic.source is \"aiDescribe\" or \"recorderAI\" and event.semantic.status is \"ready\".\n- For scroll events, preserve the recorded scroll region from event.semantic.elementDescription/replayInstruction. If the scroll happened in a specific panel, list, table, dialog body, menu, navigation area, or content pane, keep that region in the Markdown step instead of generalizing it to the whole page.\n- If event.semantic.source is \"heuristic\" or event.semantic.status is \"pending\"/\"failed\", use the screenshot/context to write the best visual instruction.\n- Coordinates are only fallback hints. Do not make coordinates the primary instruction when text or screenshots are available.\n- For a click/tap that only focuses a field before an input event, describe the target as the field/control itself. Do not target a placeholder character, typed character, caret, or inner text fragment inside the field.\n- If a target cannot be found, stop and report the missing step. Do not click similar-looking elements.\n- Screenshots are only generation-time visual evidence for you. The generated Markdown will be passed directly to agent.aiAct(markdownReplayPrompt), which accepts text only and cannot receive attached images.\n- Convert any useful screenshot evidence into textual replay instructions. Do not include screenshots, image syntax, image paths, or reference-image names in the generated Markdown.\n- Never write Markdown image syntax such as ![step context](...), reference-style images, HTML <img> tags, ./screenshots/... paths, or screenshot-* names in the output.\n\nRequired structure:\n# ${input.testName || summary.testName}\n\n## Goal\nReproduce the recorded user workflow exactly.\n\n## Target\n- Platform: ${input.target.platformId}\n- Start target: ${summary.startUrl || input.target.label || input.target.deviceId || 'Recorded target'}\n\n## Steps\n1. ...\n\nRecorder data:\n${JSON.stringify(promptPayload, null, 2)}${getMarkdownLanguageInstruction(input.language)}\n\nImportant: Return ONLY raw Markdown. Do NOT wrap the response in markdown code blocks.`;\n\n  const content: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  for (const asset of screenshotAssets) {\n    const screenshotRef = screenshotIndexByEventHash.get(asset.eventHashId);\n    content.push({\n      type: 'text',\n      text: `${screenshotRef} for event #${asset.eventIndex + 1}`,\n    });\n    content.push({\n      type: 'image',\n      url: asset.dataUrl,\n    });\n  }\n\n  return [\n    {\n      role: 'system',\n      content:\n        'You generate precise Markdown replay scripts for Midscene agent.aiAct. The final output is plain text that will be passed directly to agent.aiAct, so it must be deterministic, ordered, safe for AI execution, and must not contain image references, screenshot paths, or screenshot labels.',\n    },\n    {\n      role: 'user',\n      content,\n    },\n  ];\n}\n\nexport async function generateRecorderMarkdownReplay(\n  input: RecorderMarkdownGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> {\n  try {\n    const prompt = await createRecorderMarkdownReplayPromptForGeneration(input);\n    const response = await callAIWithStringResponse(\n      prompt,\n      createModelRuntime(modelConfig),\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return normalizeGeneratedMarkdown(response.content);\n    }\n\n    throw new Error('Failed to generate recorder Markdown replay');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder Markdown replay: ${error}`);\n  }\n}\n\nexport async function convertRecordLogIntoMarkdown(\n  log: RecorderMarkdownGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> {\n  return generateRecorderMarkdownReplay(log, modelConfig);\n}\n"],"names":["MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET","MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE","debugMarkdownReplay","getDebug","limitScreenshotAssetsForMarkdownReplay","screenshotAssets","usedPayload","asset","payloadSize","compressScreenshotAssetForMarkdownReplay","width","height","imageInfoOfBase64","longestEdge","Math","scale","dataUrl","resizeBase64ImageToJpeg","body","mimeType","parseBase64","prepareScreenshotAssetsForMarkdownReplay","compressedAssets","error","summarizeScreenshotAssets","payloadSizes","sum","size","getPromptShape","prompt","textChars","imageCount","message","content","Array","part","String","removeOmittedScreenshotPaths","summary","includedScreenshotPaths","Set","event","undefined","getMarkdownLanguageInstruction","language","normalizedLanguage","normalizeGeneratedMarkdown","trimmed","fencedMatch","createModelRuntime","modelConfig","getModelRuntime","createRecorderMarkdownReplayPrompt","input","validateEvents","rawSummary","rawScreenshotAssets","prepareRecorderGenerationContext","createRecorderMarkdownReplayPromptFromContext","createRecorderMarkdownReplayPromptForGeneration","screenshotIndexByEventHash","Map","index","events","screenshotRef","screenshotPath","eventWithoutScreenshotPath","promptPayload","getMidsceneRecorderEventDescription","promptText","stringifyMidsceneRecorderTargetBlock","JSON","generateRecorderMarkdownReplay","response","callAIWithStringResponse","Error","convertRecordLogIntoMarkdown","log"],"mappings":";;;;;;AAuBA,MAAMA,4CAA4C;AAClD,MAAMC,sCAAsC;AAC5C,MAAMC,sBAAsBC,SAAS,wBAAwB;IAC3D,SAAS;AACX;AAEA,SAASC,uCACPC,gBAA2D;IAE3D,IAAIC,cAAc;IAClB,OAAOD,iBAAiB,MAAM,CAAC,CAACE;QAC9B,MAAMC,cAAcD,MAAM,OAAO,CAAC,MAAM;QACxC,IACEC,cAAcR,6CACdM,cAAcE,cAAcR,2CAE5B,OAAO;QAETM,eAAeE;QACf,OAAO;IACT;AACF;AAEA,eAAeC,yCACbF,KAA8C;IAE9C,MAAM,EAAEG,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMC,kBAAkBL,MAAM,OAAO;IAC/D,MAAMM,cAAcC,KAAK,GAAG,CAACJ,OAAOC;IACpC,IAAIE,eAAeZ,qCACjB,OAAOM;IAGT,MAAMQ,QAAQd,sCAAsCY;IACpD,MAAMG,UAAU,MAAMC,wBAAwBV,MAAM,OAAO,EAAE;QAC3D,YAAY;YAAEG;YAAOC;QAAO;QAC5B,YAAY;YACV,OAAOG,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACJ,QAAQK;YACtC,QAAQD,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACH,SAASI;QAC1C;IACF;IACA,MAAM,EAAEG,IAAI,EAAEC,QAAQ,EAAE,GAAGC,YAAYJ;IACvC,OAAO;QACL,GAAGT,KAAK;QACRS;QACA,YAAYE;QACZC;IACF;AACF;AAEA,eAAeE,yCACbhB,gBAA2D;IAE3D,MAAMiB,mBAA8D,EAAE;IACtE,KAAK,MAAMf,SAASF,iBAClB,IAAI;QACFiB,iBAAiB,IAAI,CACnB,MAAMb,yCAAyCF;IAEnD,EAAE,OAAOgB,OAAO;QACdrB,oBACE,4DACA;YACE,aAAaK,MAAM,WAAW;YAC9B,YAAYA,MAAM,UAAU;YAC5BgB;QACF;QAEFD,iBAAiB,IAAI,CAACf;IACxB;IAEF,OAAOH,uCAAuCkB;AAChD;AAEA,SAASE,0BACPnB,gBAA2D;IAE3D,MAAMoB,eAAepB,iBAAiB,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO,CAAC,MAAM;IACzE,OAAO;QACL,OAAOF,iBAAiB,MAAM;QAC9B,mBAAmBoB,aAAa,MAAM,CAAC,CAACC,KAAKC,OAASD,MAAMC,MAAM;QAClE,iBAAiBF,aAAa,MAAM,GAAGX,KAAK,GAAG,IAAIW,gBAAgB;IACrE;AACF;AAEA,SAASG,eAAeC,MAA6B;IACnD,IAAIC,YAAY;IAChB,IAAIC,aAAa;IACjB,KAAK,MAAMC,WAAWH,OAAQ;QAC5B,MAAMI,UAAUD,QAAQ,OAAO;QAC/B,IAAI,AAAmB,YAAnB,OAAOC,SAAsB;YAC/BH,aAAaG,QAAQ,MAAM;YAC3B;QACF;QACA,IAAKC,MAAM,OAAO,CAACD,UAGnB;YAAA,KAAK,MAAME,QAAQF,QACjB,IAAI,AAAgB,YAAhB,OAAOE,QAAqBA,QAAQ,UAAUA,MAAM;gBACtD,IAAIA,AAAc,WAAdA,KAAK,IAAI,IAAe,UAAUA,MACpCL,aAAaM,OAAOD,KAAK,IAAI,EAAE,MAAM;gBAEvC,IAAIA,AAAc,YAAdA,KAAK,IAAI,EACXJ,cAAc;YAElB;QACF;IACF;IACA,OAAO;QAAED;QAAWC;IAAW;AACjC;AAEA,SAASM,6BACPC,OAAuE,EACvEjC,gBAA2D;IAE3D,MAAMkC,0BAA0B,IAAIC,IAClCnC,iBAAiB,GAAG,CAAC,CAACE,QAAUA,MAAM,YAAY;IAEpD,OAAO;QACL,GAAG+B,OAAO;QACV,QAAQA,QAAQ,MAAM,CAAC,GAAG,CAAC,CAACG,QAC1BA,MAAM,cAAc,IAAI,CAACF,wBAAwB,GAAG,CAACE,MAAM,cAAc,IACrE;gBAAE,GAAGA,KAAK;gBAAE,gBAAgBC;YAAU,IACtCD;IAER;AACF;AAEA,SAASE,+BAA+BC,QAAiB;IACvD,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;oDAE0C,EAAEA,mBAAmB;+EACM,CAAC;AAChF;AAEA,SAASC,2BAA2Bb,OAAe;IACjD,MAAMc,UAAUd,QAAQ,IAAI;IAC5B,MAAMe,cAAcD,QAAQ,KAAK,CAC/B;IAEF,OAAO,GAAIC,AAAAA,CAAAA,aAAa,CAAC,EAAE,IAAID,OAAM,EAAG,IAAI,GAAG,EAAE,CAAC;AACpD;AAEA,SAASE,mBAAmBC,WAAyB;IACnD,OAAOC,gBAAgBD;AACzB;AAEO,SAASE,mCACdC,KAAsC;IAEtCC,eAAeD,MAAM,MAAM;IAE3B,MAAM,EAAE,SAASE,UAAU,EAAE,kBAAkBC,mBAAmB,EAAE,GAClEC,iCAAiCJ;IACnC,MAAMhD,mBACJD,uCAAuCoD;IACzC,MAAMlB,UAAUD,6BAA6BkB,YAAYlD;IACzD,OAAOqD,8CACLL,OACAf,SACAjC;AAEJ;AAEA,eAAesD,gDACbN,KAAsC;IAEtCC,eAAeD,MAAM,MAAM;IAE3B,MAAM,EAAE,SAASE,UAAU,EAAE,kBAAkBC,mBAAmB,EAAE,GAClEC,iCAAiCJ;IACnC,MAAMhD,mBACJ,MAAMgB,yCAAyCmC;IACjD,MAAMlB,UAAUD,6BAA6BkB,YAAYlD;IACzD,MAAMwB,SAAS6B,8CACbL,OACAf,SACAjC;IAEFH,oBAAoB,mCAAmC;QACrD,YAAYmD,MAAM,MAAM,CAAC,MAAM;QAC/B,gBAAgBA,MAAM,cAAc;QACpC,gBAAgB7B,0BAA0BgC;QAC1C,qBAAqBhC,0BAA0BnB;QAC/C,QAAQuB,eAAeC;IACzB;IACA,OAAOA;AACT;AAEA,SAAS6B,8CACPL,KAAsC,EACtCf,OAAuE,EACvEjC,gBAA2D;IAE3D,MAAMuD,6BAA6B,IAAIC,IACrCxD,iBAAiB,GAAG,CAAC,CAACE,OAAOuD,QAAU;YACrCvD,MAAM,WAAW;YACjB,CAAC,WAAW,EAAEuD,QAAQ,GAAG;SAC1B;IAEH,MAAMC,SAASzB,QAAQ,MAAM,CAAC,GAAG,CAAC,CAACG;QACjC,MAAMuB,gBAAgBJ,2BAA2B,GAAG,CAACnB,MAAM,MAAM;QACjE,MAAM,EAAEwB,cAAc,EAAE,GAAGC,4BAA4B,GAAGzB;QAC1D,OAAOuB,gBACH;YAAE,GAAGE,0BAA0B;YAAEF;QAAc,IAC/CE;IACN;IACA,MAAMC,gBAAgB;QACpB,UAAUd,MAAM,QAAQ,IAAIf,QAAQ,QAAQ;QAC5C,QAAQ;YACN,YAAYe,MAAM,MAAM,CAAC,UAAU;YACnC,OAAOA,MAAM,MAAM,CAAC,KAAK;YACzB,QAAQA,MAAM,MAAM,CAAC,MAAM;QAC7B;QACA,UAAUf,QAAQ,QAAQ;QAC1ByB;QACA,aAAa1D,iBAAiB,GAAG,CAAC,CAACE,OAAOuD,QAAW;gBACnD,eAAe,CAAC,WAAW,EAAEA,QAAQ,GAAG;gBACxC,YAAYvD,MAAM,UAAU;gBAC5B,aAAaA,MAAM,WAAW;gBAC9B,WAAWA,MAAM,SAAS;gBAC1B,aAAa6D,oCACXf,MAAM,MAAM,CAAC9C,MAAM,UAAU,CAAC;YAElC;IACF;IACA,MAAM8D,aAAa,CAAC;;;;;;AAMtB,EAAEC,qCAAqCjB,MAAM,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;EAqBnD,EAAEA,MAAM,QAAQ,IAAIf,QAAQ,QAAQ,CAAC;;;;;;YAM3B,EAAEe,MAAM,MAAM,CAAC,UAAU,CAAC;gBACtB,EAAEf,QAAQ,QAAQ,IAAIe,MAAM,MAAM,CAAC,KAAK,IAAIA,MAAM,MAAM,CAAC,QAAQ,IAAI,kBAAkB;;;;;;AAMvG,EAAEkB,KAAK,SAAS,CAACJ,eAAe,MAAM,KAAKxB,+BAA+BU,MAAM,QAAQ,EAAE;;sFAEJ,CAAC;IAErF,MAAMpB,UAAiB;QACrB;YACE,MAAM;YACN,MAAMoC;QACR;KACD;IAED,KAAK,MAAM9D,SAASF,iBAAkB;QACpC,MAAM2D,gBAAgBJ,2BAA2B,GAAG,CAACrD,MAAM,WAAW;QACtE0B,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,GAAG+B,cAAc,YAAY,EAAEzD,MAAM,UAAU,GAAG,GAAG;QAC7D;QACA0B,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,KAAK1B,MAAM,OAAO;QACpB;IACF;IAEA,OAAO;QACL;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACN0B;QACF;KACD;AACH;AAEO,eAAeuC,+BACpBnB,KAAsC,EACtCH,WAAyB;IAEzB,IAAI;QACF,MAAMrB,SAAS,MAAM8B,gDAAgDN;QACrE,MAAMoB,WAAW,MAAMC,yBACrB7C,QACAoB,mBAAmBC;QAGrB,IAAIuB,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO3B,2BAA2B2B,SAAS,OAAO;QAGpD,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOpD,OAAO;QACd,MAAM,IAAIoD,MAAM,CAAC,6CAA6C,EAAEpD,OAAO;IACzE;AACF;AAEO,eAAeqD,6BACpBC,GAAoC,EACpC3B,WAAyB;IAEzB,OAAOsB,+BAA+BK,KAAK3B;AAC7C"}

package/dist/es/ai-model/workflows/recorder-generation/metadata.mjs

@@ -58,10 +58,8 @@ Respond with a JSON object containing exactly "title" and "description".`
         }
     ];
     for (const screenshot of screenshots)messageContent.push({
-        type: 'image_url',
-        image_url: {
-            url: screenshot
-        }
+        type: 'image',
+        url: screenshot
     });
     const response = await callAIWithObjectResponse([
         {

package/dist/es/ai-model/workflows/recorder-generation/metadata.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/recorder-generation/metadata.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/metadata.ts"],"sourcesContent":["import type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type MidsceneRecorderEvent,\n  type MidsceneRecorderTarget,\n  getMidsceneRecorderEventDescription,\n  getMidsceneRecorderScreenshotsForLLM,\n  getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\nimport { getModelRuntime } from '../../models';\nimport { callAIWithObjectResponse } from '../../service-caller';\nimport { compactRecorderSemanticForGeneration } from './common';\n\nexport interface RecorderMetadataGenerationInput {\n  target: MidsceneRecorderTarget;\n  events: MidsceneRecorderEvent[];\n  fallbackName?: string;\n  maxScreenshots?: number;\n}\n\nexport interface RecorderGeneratedMetadata {\n  title?: string;\n  description?: string;\n}\n\nfunction summarizeRecorderEvents(input: RecorderMetadataGenerationInput) {\n  const events = input.events;\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const clickEvents = events.filter((event) => event.type === 'click');\n  const inputEvents = events.filter((event) => event.type === 'input');\n  const scrollEvents = events.filter((event) => event.type === 'scroll');\n  const urls = navigationEvents\n    .map((event) => event.url)\n    .filter((url): url is string => Boolean(url));\n  const titles = navigationEvents\n    .map((event) => event.title)\n    .filter((title): title is string => Boolean(title));\n\n  return {\n    platform: input.target.platformId,\n    target: input.target,\n    fallbackName: input.fallbackName,\n    pageCount: navigationEvents.length,\n    pageTitles: titles.slice(0, 5),\n    urls: urls.slice(0, 5),\n    clickCount: clickEvents.length,\n    inputCount: inputEvents.length,\n    scrollCount: scrollEvents.length,\n    totalActions: events.length,\n    firstUrl: urls[0] || input.target.values.url || '',\n    lastUrl: urls[urls.length - 1] || '',\n    events: events.slice(0, 20).map((event) => ({\n      type: event.type,\n      actionType: event.actionType,\n      url: event.url,\n      title: event.title,\n      value: event.value,\n      description: getMidsceneRecorderEventDescription(event),\n      semantic: compactRecorderSemanticForGeneration(\n        getMidsceneRecorderSemantic(event),\n      ),\n    })),\n  };\n}\n\nfunction normalizeMetadataValue(value: unknown) {\n  return typeof value === 'string' ? value.trim() : '';\n}\n\nexport async function generateRecorderSessionMetadata(\n  input: RecorderMetadataGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<RecorderGeneratedMetadata> {\n  if (!input?.events?.length) {\n    throw new Error('generateRecorderSessionMetadata: events are required.');\n  }\n  if (!modelConfig?.modelName) {\n    throw new Error(\n      'generateRecorderSessionMetadata: modelConfig.modelName is required.',\n    );\n  }\n\n  const summary = summarizeRecorderEvents(input);\n  const screenshots = getMidsceneRecorderScreenshotsForLLM(\n    input.events,\n    input.maxScreenshots ?? 1,\n  );\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: `Generate a concise title (5-7 words) and brief description (1-2 sentences) for a Studio recording of user actions.\n\nThe recording can target Web, Android, iOS, HarmonyOS, or Computer. Do not assume it is a browser session unless the platform is web.\nDescribe what the user did or accomplished. The description should use the user as the subject, preferably starting with \"The user ...\". Do not start the description with \"The session ...\".\nThe title should be action-oriented and highlight the main task accomplished.\n\nSummary:\n${JSON.stringify(summary, null, 2)}\n\nRespond with a JSON object containing exactly \"title\" and \"description\".`,\n    },\n  ];\n\n  for (const screenshot of screenshots) {\n    messageContent.push({\n      type: 'image_url',\n      image_url: { url: screenshot },\n    });\n  }\n\n  const response = await callAIWithObjectResponse<{\n    title?: string;\n    description?: string;\n  }>(\n    [\n      {\n        role: 'system',\n        content:\n          'You generate clear, task-oriented titles and descriptions for recorded automation sessions.',\n      },\n      {\n        role: 'user',\n        content: messageContent,\n      },\n    ],\n    getModelRuntime(modelConfig),\n  );\n\n  return {\n    title: normalizeMetadataValue(response.content.title),\n    description: normalizeMetadataValue(response.content.description),\n  };\n}\n"],"names":["summarizeRecorderEvents","input","events","navigationEvents","event","clickEvents","inputEvents","scrollEvents","urls","url","Boolean","titles","title","getMidsceneRecorderEventDescription","compactRecorderSemanticForGeneration","getMidsceneRecorderSemantic","normalizeMetadataValue","value","generateRecorderSessionMetadata","modelConfig","Error","summary","screenshots","getMidsceneRecorderScreenshotsForLLM","messageContent","JSON","screenshot","response","callAIWithObjectResponse","getModelRuntime"],"mappings":";;;;AAwBA,SAASA,wBAAwBC,KAAsC;IACrE,MAAMC,SAASD,MAAM,MAAM;IAC3B,MAAME,mBAAmBD,OAAO,MAAM,CACpC,CAACE,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,cAAcH,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAME,cAAcJ,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAMG,eAAeL,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACxD,MAAMI,OAAOL,iBACV,GAAG,CAAC,CAACC,QAAUA,MAAM,GAAG,EACxB,MAAM,CAAC,CAACK,MAAuBC,QAAQD;IAC1C,MAAME,SAASR,iBACZ,GAAG,CAAC,CAACC,QAAUA,MAAM,KAAK,EAC1B,MAAM,CAAC,CAACQ,QAA2BF,QAAQE;IAE9C,OAAO;QACL,UAAUX,MAAM,MAAM,CAAC,UAAU;QACjC,QAAQA,MAAM,MAAM;QACpB,cAAcA,MAAM,YAAY;QAChC,WAAWE,iBAAiB,MAAM;QAClC,YAAYQ,OAAO,KAAK,CAAC,GAAG;QAC5B,MAAMH,KAAK,KAAK,CAAC,GAAG;QACpB,YAAYH,YAAY,MAAM;QAC9B,YAAYC,YAAY,MAAM;QAC9B,aAAaC,aAAa,MAAM;QAChC,cAAcL,OAAO,MAAM;QAC3B,UAAUM,IAAI,CAAC,EAAE,IAAIP,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI;QAChD,SAASO,IAAI,CAACA,KAAK,MAAM,GAAG,EAAE,IAAI;QAClC,QAAQN,OAAO,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAACE,QAAW;gBAC1C,MAAMA,MAAM,IAAI;gBAChB,YAAYA,MAAM,UAAU;gBAC5B,KAAKA,MAAM,GAAG;gBACd,OAAOA,MAAM,KAAK;gBAClB,OAAOA,MAAM,KAAK;gBAClB,aAAaS,oCAAoCT;gBACjD,UAAUU,qCACRC,4BAA4BX;YAEhC;IACF;AACF;AAEA,SAASY,uBAAuBC,KAAc;IAC5C,OAAO,AAAiB,YAAjB,OAAOA,QAAqBA,MAAM,IAAI,KAAK;AACpD;AAEO,eAAeC,gCACpBjB,KAAsC,EACtCkB,WAAyB;IAEzB,IAAI,CAAClB,OAAO,QAAQ,QAClB,MAAM,IAAImB,MAAM;IAElB,IAAI,CAACD,aAAa,WAChB,MAAM,IAAIC,MACR;IAIJ,MAAMC,UAAUrB,wBAAwBC;IACxC,MAAMqB,cAAcC,qCAClBtB,MAAM,MAAM,EACZA,MAAM,cAAc,IAAI;IAE1B,MAAMuB,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAM,CAAC;;;;;;;AAOb,EAAEC,KAAK,SAAS,CAACJ,SAAS,MAAM,GAAG;;wEAEqC,CAAC;QACrE;KACD;IAED,KAAK,MAAMK,cAAcJ,YACvBE,eAAe,IAAI,CAAC;QAClB,MAAM;QACN,WAAW;YAAE,KAAKE;QAAW;IAC/B;IAGF,MAAMC,WAAW,MAAMC,yBAIrB;QACE;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACN,SAASJ;QACX;KACD,EACDK,gBAAgBV;IAGlB,OAAO;QACL,OAAOH,uBAAuBW,SAAS,OAAO,CAAC,KAAK;QACpD,aAAaX,uBAAuBW,SAAS,OAAO,CAAC,WAAW;IAClE;AACF"}
+{"version":3,"file":"ai-model/workflows/recorder-generation/metadata.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/metadata.ts"],"sourcesContent":["import type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type MidsceneRecorderEvent,\n  type MidsceneRecorderTarget,\n  getMidsceneRecorderEventDescription,\n  getMidsceneRecorderScreenshotsForLLM,\n  getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\nimport { getModelRuntime } from '../../models';\nimport { callAIWithObjectResponse } from '../../service-caller';\nimport type { MessageContent } from '../../service-caller/types';\nimport { compactRecorderSemanticForGeneration } from './common';\n\nexport interface RecorderMetadataGenerationInput {\n  target: MidsceneRecorderTarget;\n  events: MidsceneRecorderEvent[];\n  fallbackName?: string;\n  maxScreenshots?: number;\n}\n\nexport interface RecorderGeneratedMetadata {\n  title?: string;\n  description?: string;\n}\n\nfunction summarizeRecorderEvents(input: RecorderMetadataGenerationInput) {\n  const events = input.events;\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const clickEvents = events.filter((event) => event.type === 'click');\n  const inputEvents = events.filter((event) => event.type === 'input');\n  const scrollEvents = events.filter((event) => event.type === 'scroll');\n  const urls = navigationEvents\n    .map((event) => event.url)\n    .filter((url): url is string => Boolean(url));\n  const titles = navigationEvents\n    .map((event) => event.title)\n    .filter((title): title is string => Boolean(title));\n\n  return {\n    platform: input.target.platformId,\n    target: input.target,\n    fallbackName: input.fallbackName,\n    pageCount: navigationEvents.length,\n    pageTitles: titles.slice(0, 5),\n    urls: urls.slice(0, 5),\n    clickCount: clickEvents.length,\n    inputCount: inputEvents.length,\n    scrollCount: scrollEvents.length,\n    totalActions: events.length,\n    firstUrl: urls[0] || input.target.values.url || '',\n    lastUrl: urls[urls.length - 1] || '',\n    events: events.slice(0, 20).map((event) => ({\n      type: event.type,\n      actionType: event.actionType,\n      url: event.url,\n      title: event.title,\n      value: event.value,\n      description: getMidsceneRecorderEventDescription(event),\n      semantic: compactRecorderSemanticForGeneration(\n        getMidsceneRecorderSemantic(event),\n      ),\n    })),\n  };\n}\n\nfunction normalizeMetadataValue(value: unknown) {\n  return typeof value === 'string' ? value.trim() : '';\n}\n\nexport async function generateRecorderSessionMetadata(\n  input: RecorderMetadataGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<RecorderGeneratedMetadata> {\n  if (!input?.events?.length) {\n    throw new Error('generateRecorderSessionMetadata: events are required.');\n  }\n  if (!modelConfig?.modelName) {\n    throw new Error(\n      'generateRecorderSessionMetadata: modelConfig.modelName is required.',\n    );\n  }\n\n  const summary = summarizeRecorderEvents(input);\n  const screenshots = getMidsceneRecorderScreenshotsForLLM(\n    input.events,\n    input.maxScreenshots ?? 1,\n  );\n  const messageContent: MessageContent[] = [\n    {\n      type: 'text',\n      text: `Generate a concise title (5-7 words) and brief description (1-2 sentences) for a Studio recording of user actions.\n\nThe recording can target Web, Android, iOS, HarmonyOS, or Computer. Do not assume it is a browser session unless the platform is web.\nDescribe what the user did or accomplished. The description should use the user as the subject, preferably starting with \"The user ...\". Do not start the description with \"The session ...\".\nThe title should be action-oriented and highlight the main task accomplished.\n\nSummary:\n${JSON.stringify(summary, null, 2)}\n\nRespond with a JSON object containing exactly \"title\" and \"description\".`,\n    },\n  ];\n\n  for (const screenshot of screenshots) {\n    messageContent.push({\n      type: 'image',\n      url: screenshot,\n    });\n  }\n\n  const response = await callAIWithObjectResponse<{\n    title?: string;\n    description?: string;\n  }>(\n    [\n      {\n        role: 'system',\n        content:\n          'You generate clear, task-oriented titles and descriptions for recorded automation sessions.',\n      },\n      {\n        role: 'user',\n        content: messageContent,\n      },\n    ],\n    getModelRuntime(modelConfig),\n  );\n\n  return {\n    title: normalizeMetadataValue(response.content.title),\n    description: normalizeMetadataValue(response.content.description),\n  };\n}\n"],"names":["summarizeRecorderEvents","input","events","navigationEvents","event","clickEvents","inputEvents","scrollEvents","urls","url","Boolean","titles","title","getMidsceneRecorderEventDescription","compactRecorderSemanticForGeneration","getMidsceneRecorderSemantic","normalizeMetadataValue","value","generateRecorderSessionMetadata","modelConfig","Error","summary","screenshots","getMidsceneRecorderScreenshotsForLLM","messageContent","JSON","screenshot","response","callAIWithObjectResponse","getModelRuntime"],"mappings":";;;;AAyBA,SAASA,wBAAwBC,KAAsC;IACrE,MAAMC,SAASD,MAAM,MAAM;IAC3B,MAAME,mBAAmBD,OAAO,MAAM,CACpC,CAACE,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,cAAcH,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAME,cAAcJ,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAMG,eAAeL,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACxD,MAAMI,OAAOL,iBACV,GAAG,CAAC,CAACC,QAAUA,MAAM,GAAG,EACxB,MAAM,CAAC,CAACK,MAAuBC,QAAQD;IAC1C,MAAME,SAASR,iBACZ,GAAG,CAAC,CAACC,QAAUA,MAAM,KAAK,EAC1B,MAAM,CAAC,CAACQ,QAA2BF,QAAQE;IAE9C,OAAO;QACL,UAAUX,MAAM,MAAM,CAAC,UAAU;QACjC,QAAQA,MAAM,MAAM;QACpB,cAAcA,MAAM,YAAY;QAChC,WAAWE,iBAAiB,MAAM;QAClC,YAAYQ,OAAO,KAAK,CAAC,GAAG;QAC5B,MAAMH,KAAK,KAAK,CAAC,GAAG;QACpB,YAAYH,YAAY,MAAM;QAC9B,YAAYC,YAAY,MAAM;QAC9B,aAAaC,aAAa,MAAM;QAChC,cAAcL,OAAO,MAAM;QAC3B,UAAUM,IAAI,CAAC,EAAE,IAAIP,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI;QAChD,SAASO,IAAI,CAACA,KAAK,MAAM,GAAG,EAAE,IAAI;QAClC,QAAQN,OAAO,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAACE,QAAW;gBAC1C,MAAMA,MAAM,IAAI;gBAChB,YAAYA,MAAM,UAAU;gBAC5B,KAAKA,MAAM,GAAG;gBACd,OAAOA,MAAM,KAAK;gBAClB,OAAOA,MAAM,KAAK;gBAClB,aAAaS,oCAAoCT;gBACjD,UAAUU,qCACRC,4BAA4BX;YAEhC;IACF;AACF;AAEA,SAASY,uBAAuBC,KAAc;IAC5C,OAAO,AAAiB,YAAjB,OAAOA,QAAqBA,MAAM,IAAI,KAAK;AACpD;AAEO,eAAeC,gCACpBjB,KAAsC,EACtCkB,WAAyB;IAEzB,IAAI,CAAClB,OAAO,QAAQ,QAClB,MAAM,IAAImB,MAAM;IAElB,IAAI,CAACD,aAAa,WAChB,MAAM,IAAIC,MACR;IAIJ,MAAMC,UAAUrB,wBAAwBC;IACxC,MAAMqB,cAAcC,qCAClBtB,MAAM,MAAM,EACZA,MAAM,cAAc,IAAI;IAE1B,MAAMuB,iBAAmC;QACvC;YACE,MAAM;YACN,MAAM,CAAC;;;;;;;AAOb,EAAEC,KAAK,SAAS,CAACJ,SAAS,MAAM,GAAG;;wEAEqC,CAAC;QACrE;KACD;IAED,KAAK,MAAMK,cAAcJ,YACvBE,eAAe,IAAI,CAAC;QAClB,MAAM;QACN,KAAKE;IACP;IAGF,MAAMC,WAAW,MAAMC,yBAIrB;QACE;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACN,SAASJ;QACX;KACD,EACDK,gBAAgBV;IAGlB,OAAO;QACL,OAAOH,uBAAuBW,SAAS,OAAO,CAAC,KAAK;QACpD,aAAaX,uBAAuBW,SAAS,OAAO,CAAC,WAAW;IAClE;AACF"}

package/dist/es/ai-model/workflows/recorder-generation/playwright.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/recorder-generation/playwright.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/playwright.ts"],"sourcesContent":["import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { getModelRuntime } from '../../models';\nimport { callAI, callAIWithStringResponse } from '../../service-caller';\n// Import shared utilities and types from yaml generation.\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './common';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAI(prompt, modelRuntime, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["createModelRuntime","modelConfig","getModelRuntime","generatePlaywrightTest","events","options","modelRuntime","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","Error","generatePlaywrightTestStream","callAI"],"mappings":";;;;AA2DA,SAASA,mBAAmBC,WAAyB;IACnD,OAAOC,gBAAgBD;AACzB;AAKO,MAAME,yBAAyB,OACpCC,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;8LAWiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,yBAAyBF,QAAQb;IAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIE,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CnB,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;8LAYiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMmB,OAAOL,QAAQb,cAAc;QACxC,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,yBAAyBF,QAAQb;QAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB;AACF"}
+{"version":3,"file":"ai-model/workflows/recorder-generation/playwright.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/playwright.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getModelRuntime } from '../../models';\nimport { callAI, callAIWithStringResponse } from '../../service-caller';\n// Import shared utilities and types from yaml generation.\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './common';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAI(prompt, modelRuntime, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["createModelRuntime","modelConfig","getModelRuntime","generatePlaywrightTest","events","options","modelRuntime","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","Error","generatePlaywrightTestStream","callAI"],"mappings":";;;;AA2DA,SAASA,mBAAmBC,WAAyB;IACnD,OAAOC,gBAAgBD;AACzB;AAKO,MAAME,yBAAyB,OACpCC,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;8LAWiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,yBAAyBF,QAAQb;IAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIE,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CnB,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;8LAYiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMmB,OAAOL,QAAQb,cAAc;QACxC,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,yBAAyBF,QAAQb;QAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB;AACF"}

package/dist/es/ai-model/workflows/recorder-generation/yaml.mjs

@@ -66,10 +66,8 @@ Important: Return ONLY the raw YAML content. Do NOT wrap the response in markdow
                         text: `Screenshot asset for event #${asset.eventIndex + 1}: ${asset.relativePath}`
                     },
                     {
-                        type: 'image_url',
-                        image_url: {
-                            url: asset.dataUrl
-                        }
+                        type: 'image',
+                        url: asset.dataUrl
                     }
                 ])
         });

package/dist/es/ai-model/workflows/recorder-generation/yaml.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/recorder-generation/yaml.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/yaml.ts"],"sourcesContent":["import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  type MidsceneRecorderTarget,\n  stringifyMidsceneRecorderTargetBlock,\n} from '@midscene/shared/recorder';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { getModelRuntime } from '../../models';\nimport { callAI, callAIWithStringResponse } from '../../service-caller';\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type FilteredEvents,\n  type InputDescription,\n  type ProcessedEvent,\n  type RecorderGenerationContext,\n  type RecorderGenerationInput,\n  type RecorderGenerationOptions,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  prepareRecorderGenerationContext,\n  processEventsForLLM,\n  validateEvents,\n} from './common';\n\nexport type YamlGenerationOptions = RecorderGenerationOptions;\nexport type RecorderYamlGenerationInput = RecorderGenerationInput;\n\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  EventSummary,\n  FilteredEvents,\n  InputDescription,\n  ProcessedEvent,\n  RecorderGenerationContext,\n};\n\nexport {\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  prepareRecorderGenerationContext,\n  processEventsForLLM,\n  validateEvents,\n};\n\nconst getYamlLanguageInstruction = (language?: string) => {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable YAML content in ${normalizedLanguage}.\n- Keep YAML keys, field names, and Midscene API names unchanged.`;\n};\n\nconst createYamlPrompt = ({\n  yamlSummary,\n  screenshotAssets,\n  language,\n  targetBlock,\n  target,\n}: {\n  yamlSummary: EventSummary & { includeTimestamps: boolean };\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n  language?: string;\n  targetBlock: string;\n  target: MidsceneRecorderTarget;\n}): ChatCompletionMessageParam[] => {\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n    },\n    {\n      role: 'user',\n      content: `Generate YAML test for Midscene.js automation from recorded events.\n\nTarget platform:\n- Preserve this exact top-level target platform: ${target.platformId}\n- Use exactly one top-level target block.\n- The target block must be:\n${targetBlock}\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nScreenshot assets:\n${JSON.stringify(\n  screenshotAssets.map((asset) => ({\n    eventIndex: asset.eventIndex,\n    eventHashId: asset.eventHashId,\n    eventType: asset.eventType,\n    relativePath: asset.relativePath,\n    description: yamlSummary.events[asset.eventIndex]?.description,\n  })),\n  null,\n  2,\n)}\n\nConvert events:\n- navigation → target URL or aiAction only when the target platform supports it\n- click → aiTap with the semantic element description\n- input → aiInput with value and semantic locate\n- scroll → aiScroll with appropriate direction and semantic scroll area\n- keydown → aiKeyboardPress\n- Add aiAssert for important state changes\n- Prefer event.semantic.replayInstruction and event.semantic.elementDescription when event.semantic.source is \"aiDescribe\" or \"recorderAI\" and event.semantic.status is \"ready\".\n- If event.semantic.source is \"heuristic\" or event.semantic.status is \"pending\"/\"failed\", use the screenshot/context to write the best visual instruction, and avoid raw coordinates unless there is no reliable semantic description.\n- Screenshot assets are context only. Use their eventIndex/eventHashId relationship to understand the matching event, but do not include screenshot file paths in the YAML unless the Midscene YAML API explicitly needs them.${getYamlLanguageInstruction(language)}\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n    },\n  ];\n\n  if (screenshotAssets.length > 0) {\n    prompt.push({\n      role: 'user',\n      content:\n        'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    prompt.push({\n      role: 'user',\n      content: screenshotAssets.flatMap((asset) => [\n        {\n          type: 'text',\n          text: `Screenshot asset for event #${asset.eventIndex + 1}: ${asset.relativePath}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: asset.dataUrl,\n          },\n        },\n      ]),\n    });\n  }\n\n  return prompt;\n};\n\nfunction createDefaultWebTarget(\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n): MidsceneRecorderTarget {\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const firstUrl =\n    options.navigationInfo?.urls?.find(Boolean) ||\n    navigationEvents.find((event) => event.url)?.url ||\n    '';\n  const firstViewport =\n    options.navigationInfo?.initialViewport ||\n    events.find((event) => event.pageInfo)?.pageInfo;\n\n  return {\n    platformId: 'web',\n    deviceId: firstUrl || undefined,\n    label: firstUrl || 'Web',\n    values: {\n      url: firstUrl,\n      ...(firstViewport?.width ? { viewportWidth: firstViewport.width } : {}),\n      ...(firstViewport?.height\n        ? { viewportHeight: firstViewport.height }\n        : {}),\n    },\n  };\n}\n\nfunction normalizeGeneratedYaml(content: string) {\n  const trimmed = content.trim();\n  const fencedMatch = trimmed.match(/^```(?:ya?ml)?\\s*([\\s\\S]*?)\\s*```$/i);\n  return `${(fencedMatch?.[1] ?? trimmed).trim()}\\n`;\n}\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\nfunction createRecorderYamlPrompt(\n  input: RecorderYamlGenerationInput,\n): ChatCompletionMessageParam[] {\n  const { summary, screenshotAssets } = prepareRecorderGenerationContext(input);\n  const yamlSummary = {\n    ...summary,\n    target: input.target,\n    includeTimestamps: input.includeTimestamps || false,\n  };\n\n  return createYamlPrompt({\n    yamlSummary,\n    screenshotAssets,\n    language: input.language,\n    target: input.target,\n    targetBlock: stringifyMidsceneRecorderTargetBlock(input.target),\n  });\n}\n\nexport const generateRecorderYamlTest = async (\n  input: RecorderYamlGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  try {\n    const prompt = createRecorderYamlPrompt(input);\n    const response = await callAIWithStringResponse(\n      prompt,\n      createModelRuntime(modelConfig),\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return normalizeGeneratedYaml(response.content);\n    }\n\n    throw new Error('Failed to generate recorder YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder YAML test: ${error}`);\n  }\n};\n\nexport const generateRecorderYamlTestStream = async (\n  input: RecorderYamlGenerationInput,\n  options: StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  try {\n    const prompt = createRecorderYamlPrompt(input);\n    const modelRuntime = createModelRuntime(modelConfig);\n    if (options.stream && options.onChunk) {\n      return await callAI(prompt, modelRuntime, {\n        stream: true,\n        onChunk: options.onChunk,\n      });\n    }\n\n    const response = await callAIWithStringResponse(prompt, modelRuntime);\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: normalizeGeneratedYaml(response.content),\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate recorder YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder YAML test: ${error}`);\n  }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  return generateRecorderYamlTest(\n    {\n      ...options,\n      target: createDefaultWebTarget(events, options),\n      events,\n    },\n    modelConfig,\n  );\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  return generateRecorderYamlTestStream(\n    {\n      ...options,\n      target: createDefaultWebTarget(events, options),\n      events,\n    },\n    options,\n    modelConfig,\n  );\n};\n"],"names":["getYamlLanguageInstruction","language","normalizedLanguage","createYamlPrompt","yamlSummary","screenshotAssets","targetBlock","target","prompt","YAML_EXAMPLE_CODE","JSON","asset","createDefaultWebTarget","events","options","navigationEvents","event","firstUrl","Boolean","firstViewport","undefined","normalizeGeneratedYaml","content","trimmed","fencedMatch","createModelRuntime","modelConfig","getModelRuntime","createRecorderYamlPrompt","input","summary","prepareRecorderGenerationContext","stringifyMidsceneRecorderTargetBlock","generateRecorderYamlTest","response","callAIWithStringResponse","Error","error","generateRecorderYamlTestStream","modelRuntime","callAI","generateYamlTest","generateYamlTestStream"],"mappings":";;;;;AA4DA,MAAMA,6BAA6B,CAACC;IAClC,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;2CAEiC,EAAEA,mBAAmB;gEACA,CAAC;AACjE;AAEA,MAAMC,mBAAmB,CAAC,EACxBC,WAAW,EACXC,gBAAgB,EAChBJ,QAAQ,EACRK,WAAW,EACXC,MAAM,EAOP;IACC,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAAS,CAAC,4GAA4G,EAAEC,mBAAmB;QAC7I;QACA;YACE,MAAM;YACN,SAAS,CAAC;;;iDAGiC,EAAEF,OAAO,UAAU,CAAC;;;AAGrE,EAAED,YAAY;;;AAGd,EAAEI,KAAK,SAAS,CAACN,aAAa,MAAM,GAAG;;;AAGvC,EAAEM,KAAK,SAAS,CACdL,iBAAiB,GAAG,CAAC,CAACM,QAAW;oBAC/B,YAAYA,MAAM,UAAU;oBAC5B,aAAaA,MAAM,WAAW;oBAC9B,WAAWA,MAAM,SAAS;oBAC1B,cAAcA,MAAM,YAAY;oBAChC,aAAaP,YAAY,MAAM,CAACO,MAAM,UAAU,CAAC,EAAE;gBACrD,KACA,MACA,GACA;;;;;;;;;;;8NAW4N,EAAEX,2BAA2BC,UAAU;;8JAEvG,CAAC;QAC3J;KACD;IAED,IAAII,iBAAiB,MAAM,GAAG,GAAG;QAC/BG,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SACE;QACJ;QAEAA,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SAASH,iBAAiB,OAAO,CAAC,CAACM,QAAU;oBAC3C;wBACE,MAAM;wBACN,MAAM,CAAC,4BAA4B,EAAEA,MAAM,UAAU,GAAG,EAAE,EAAE,EAAEA,MAAM,YAAY,EAAE;oBACpF;oBACA;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKA,MAAM,OAAO;wBACpB;oBACF;iBACD;QACH;IACF;IAEA,OAAOH;AACT;AAEA,SAASI,uBACPC,MAA6B,EAC7BC,OAA8B;IAE9B,MAAMC,mBAAmBF,OAAO,MAAM,CACpC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,WACJH,QAAQ,cAAc,EAAE,MAAM,KAAKI,YACnCH,iBAAiB,IAAI,CAAC,CAACC,QAAUA,MAAM,GAAG,GAAG,OAC7C;IACF,MAAMG,gBACJL,QAAQ,cAAc,EAAE,mBACxBD,OAAO,IAAI,CAAC,CAACG,QAAUA,MAAM,QAAQ,GAAG;IAE1C,OAAO;QACL,YAAY;QACZ,UAAUC,YAAYG;QACtB,OAAOH,YAAY;QACnB,QAAQ;YACN,KAAKA;YACL,GAAIE,eAAe,QAAQ;gBAAE,eAAeA,cAAc,KAAK;YAAC,IAAI,CAAC,CAAC;YACtE,GAAIA,eAAe,SACf;gBAAE,gBAAgBA,cAAc,MAAM;YAAC,IACvC,CAAC,CAAC;QACR;IACF;AACF;AAEA,SAASE,uBAAuBC,OAAe;IAC7C,MAAMC,UAAUD,QAAQ,IAAI;IAC5B,MAAME,cAAcD,QAAQ,KAAK,CAAC;IAClC,OAAO,GAAIC,AAAAA,CAAAA,aAAa,CAAC,EAAE,IAAID,OAAM,EAAG,IAAI,GAAG,EAAE,CAAC;AACpD;AAEA,SAASE,mBAAmBC,WAAyB;IACnD,OAAOC,gBAAgBD;AACzB;AAEA,SAASE,yBACPC,KAAkC;IAElC,MAAM,EAAEC,OAAO,EAAEzB,gBAAgB,EAAE,GAAG0B,iCAAiCF;IACvE,MAAMzB,cAAc;QAClB,GAAG0B,OAAO;QACV,QAAQD,MAAM,MAAM;QACpB,mBAAmBA,MAAM,iBAAiB,IAAI;IAChD;IAEA,OAAO1B,iBAAiB;QACtBC;QACAC;QACA,UAAUwB,MAAM,QAAQ;QACxB,QAAQA,MAAM,MAAM;QACpB,aAAaG,qCAAqCH,MAAM,MAAM;IAChE;AACF;AAEO,MAAMI,2BAA2B,OACtCJ,OACAH;IAEA,IAAI;QACF,MAAMlB,SAASoB,yBAAyBC;QACxC,MAAMK,WAAW,MAAMC,yBACrB3B,QACAiB,mBAAmBC;QAGrB,IAAIQ,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOb,uBAAuBa,SAAS,OAAO;QAGhD,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOC,OAAO;QACd,MAAM,IAAID,MAAM,CAAC,uCAAuC,EAAEC,OAAO;IACnE;AACF;AAEO,MAAMC,iCAAiC,OAC5CT,OACAf,SACAY;IAEA,IAAI;QACF,MAAMlB,SAASoB,yBAAyBC;QACxC,MAAMU,eAAed,mBAAmBC;QACxC,IAAIZ,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EACnC,OAAO,MAAM0B,OAAOhC,QAAQ+B,cAAc;YACxC,QAAQ;YACR,SAASzB,QAAQ,OAAO;QAC1B;QAGF,MAAMoB,WAAW,MAAMC,yBAAyB3B,QAAQ+B;QACxD,IAAIL,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASb,uBAAuBa,SAAS,OAAO;YAChD,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOC,OAAO;QACd,MAAM,IAAID,MAAM,CAAC,uCAAuC,EAAEC,OAAO;IACnE;AACF;AAOO,MAAMI,mBAAmB,OAC9B5B,QACAC,SACAY,cAEOO,yBACL;QACE,GAAGnB,OAAO;QACV,QAAQF,uBAAuBC,QAAQC;QACvCD;IACF,GACAa;AAOG,MAAMgB,yBAAyB,OACpC7B,QACAC,SACAY,cAEOY,+BACL;QACE,GAAGxB,OAAO;QACV,QAAQF,uBAAuBC,QAAQC;QACvCD;IACF,GACAC,SACAY"}
+{"version":3,"file":"ai-model/workflows/recorder-generation/yaml.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/yaml.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  type MidsceneRecorderTarget,\n  stringifyMidsceneRecorderTargetBlock,\n} from '@midscene/shared/recorder';\nimport { getModelRuntime } from '../../models';\nimport { callAI, callAIWithStringResponse } from '../../service-caller';\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type FilteredEvents,\n  type InputDescription,\n  type ProcessedEvent,\n  type RecorderGenerationContext,\n  type RecorderGenerationInput,\n  type RecorderGenerationOptions,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  prepareRecorderGenerationContext,\n  processEventsForLLM,\n  validateEvents,\n} from './common';\n\nexport type YamlGenerationOptions = RecorderGenerationOptions;\nexport type RecorderYamlGenerationInput = RecorderGenerationInput;\n\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  EventSummary,\n  FilteredEvents,\n  InputDescription,\n  ProcessedEvent,\n  RecorderGenerationContext,\n};\n\nexport {\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  prepareRecorderGenerationContext,\n  processEventsForLLM,\n  validateEvents,\n};\n\nconst getYamlLanguageInstruction = (language?: string) => {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable YAML content in ${normalizedLanguage}.\n- Keep YAML keys, field names, and Midscene API names unchanged.`;\n};\n\nconst createYamlPrompt = ({\n  yamlSummary,\n  screenshotAssets,\n  language,\n  targetBlock,\n  target,\n}: {\n  yamlSummary: EventSummary & { includeTimestamps: boolean };\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n  language?: string;\n  targetBlock: string;\n  target: MidsceneRecorderTarget;\n}): ConversationMessage[] => {\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n    },\n    {\n      role: 'user',\n      content: `Generate YAML test for Midscene.js automation from recorded events.\n\nTarget platform:\n- Preserve this exact top-level target platform: ${target.platformId}\n- Use exactly one top-level target block.\n- The target block must be:\n${targetBlock}\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nScreenshot assets:\n${JSON.stringify(\n  screenshotAssets.map((asset) => ({\n    eventIndex: asset.eventIndex,\n    eventHashId: asset.eventHashId,\n    eventType: asset.eventType,\n    relativePath: asset.relativePath,\n    description: yamlSummary.events[asset.eventIndex]?.description,\n  })),\n  null,\n  2,\n)}\n\nConvert events:\n- navigation → target URL or aiAction only when the target platform supports it\n- click → aiTap with the semantic element description\n- input → aiInput with value and semantic locate\n- scroll → aiScroll with appropriate direction and semantic scroll area\n- keydown → aiKeyboardPress\n- Add aiAssert for important state changes\n- Prefer event.semantic.replayInstruction and event.semantic.elementDescription when event.semantic.source is \"aiDescribe\" or \"recorderAI\" and event.semantic.status is \"ready\".\n- If event.semantic.source is \"heuristic\" or event.semantic.status is \"pending\"/\"failed\", use the screenshot/context to write the best visual instruction, and avoid raw coordinates unless there is no reliable semantic description.\n- Screenshot assets are context only. Use their eventIndex/eventHashId relationship to understand the matching event, but do not include screenshot file paths in the YAML unless the Midscene YAML API explicitly needs them.${getYamlLanguageInstruction(language)}\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n    },\n  ];\n\n  if (screenshotAssets.length > 0) {\n    prompt.push({\n      role: 'user',\n      content:\n        'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    prompt.push({\n      role: 'user',\n      content: screenshotAssets.flatMap((asset) => [\n        {\n          type: 'text',\n          text: `Screenshot asset for event #${asset.eventIndex + 1}: ${asset.relativePath}`,\n        },\n        {\n          type: 'image',\n          url: asset.dataUrl,\n        },\n      ]),\n    });\n  }\n\n  return prompt;\n};\n\nfunction createDefaultWebTarget(\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n): MidsceneRecorderTarget {\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const firstUrl =\n    options.navigationInfo?.urls?.find(Boolean) ||\n    navigationEvents.find((event) => event.url)?.url ||\n    '';\n  const firstViewport =\n    options.navigationInfo?.initialViewport ||\n    events.find((event) => event.pageInfo)?.pageInfo;\n\n  return {\n    platformId: 'web',\n    deviceId: firstUrl || undefined,\n    label: firstUrl || 'Web',\n    values: {\n      url: firstUrl,\n      ...(firstViewport?.width ? { viewportWidth: firstViewport.width } : {}),\n      ...(firstViewport?.height\n        ? { viewportHeight: firstViewport.height }\n        : {}),\n    },\n  };\n}\n\nfunction normalizeGeneratedYaml(content: string) {\n  const trimmed = content.trim();\n  const fencedMatch = trimmed.match(/^```(?:ya?ml)?\\s*([\\s\\S]*?)\\s*```$/i);\n  return `${(fencedMatch?.[1] ?? trimmed).trim()}\\n`;\n}\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\nfunction createRecorderYamlPrompt(\n  input: RecorderYamlGenerationInput,\n): ConversationMessage[] {\n  const { summary, screenshotAssets } = prepareRecorderGenerationContext(input);\n  const yamlSummary = {\n    ...summary,\n    target: input.target,\n    includeTimestamps: input.includeTimestamps || false,\n  };\n\n  return createYamlPrompt({\n    yamlSummary,\n    screenshotAssets,\n    language: input.language,\n    target: input.target,\n    targetBlock: stringifyMidsceneRecorderTargetBlock(input.target),\n  });\n}\n\nexport const generateRecorderYamlTest = async (\n  input: RecorderYamlGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  try {\n    const prompt = createRecorderYamlPrompt(input);\n    const response = await callAIWithStringResponse(\n      prompt,\n      createModelRuntime(modelConfig),\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return normalizeGeneratedYaml(response.content);\n    }\n\n    throw new Error('Failed to generate recorder YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder YAML test: ${error}`);\n  }\n};\n\nexport const generateRecorderYamlTestStream = async (\n  input: RecorderYamlGenerationInput,\n  options: StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  try {\n    const prompt = createRecorderYamlPrompt(input);\n    const modelRuntime = createModelRuntime(modelConfig);\n    if (options.stream && options.onChunk) {\n      return await callAI(prompt, modelRuntime, {\n        stream: true,\n        onChunk: options.onChunk,\n      });\n    }\n\n    const response = await callAIWithStringResponse(prompt, modelRuntime);\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: normalizeGeneratedYaml(response.content),\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate recorder YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder YAML test: ${error}`);\n  }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  return generateRecorderYamlTest(\n    {\n      ...options,\n      target: createDefaultWebTarget(events, options),\n      events,\n    },\n    modelConfig,\n  );\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  return generateRecorderYamlTestStream(\n    {\n      ...options,\n      target: createDefaultWebTarget(events, options),\n      events,\n    },\n    options,\n    modelConfig,\n  );\n};\n"],"names":["getYamlLanguageInstruction","language","normalizedLanguage","createYamlPrompt","yamlSummary","screenshotAssets","targetBlock","target","prompt","YAML_EXAMPLE_CODE","JSON","asset","createDefaultWebTarget","events","options","navigationEvents","event","firstUrl","Boolean","firstViewport","undefined","normalizeGeneratedYaml","content","trimmed","fencedMatch","createModelRuntime","modelConfig","getModelRuntime","createRecorderYamlPrompt","input","summary","prepareRecorderGenerationContext","stringifyMidsceneRecorderTargetBlock","generateRecorderYamlTest","response","callAIWithStringResponse","Error","error","generateRecorderYamlTestStream","modelRuntime","callAI","generateYamlTest","generateYamlTestStream"],"mappings":";;;;;AA4DA,MAAMA,6BAA6B,CAACC;IAClC,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;2CAEiC,EAAEA,mBAAmB;gEACA,CAAC;AACjE;AAEA,MAAMC,mBAAmB,CAAC,EACxBC,WAAW,EACXC,gBAAgB,EAChBJ,QAAQ,EACRK,WAAW,EACXC,MAAM,EAOP;IACC,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAAS,CAAC,4GAA4G,EAAEC,mBAAmB;QAC7I;QACA;YACE,MAAM;YACN,SAAS,CAAC;;;iDAGiC,EAAEF,OAAO,UAAU,CAAC;;;AAGrE,EAAED,YAAY;;;AAGd,EAAEI,KAAK,SAAS,CAACN,aAAa,MAAM,GAAG;;;AAGvC,EAAEM,KAAK,SAAS,CACdL,iBAAiB,GAAG,CAAC,CAACM,QAAW;oBAC/B,YAAYA,MAAM,UAAU;oBAC5B,aAAaA,MAAM,WAAW;oBAC9B,WAAWA,MAAM,SAAS;oBAC1B,cAAcA,MAAM,YAAY;oBAChC,aAAaP,YAAY,MAAM,CAACO,MAAM,UAAU,CAAC,EAAE;gBACrD,KACA,MACA,GACA;;;;;;;;;;;8NAW4N,EAAEX,2BAA2BC,UAAU;;8JAEvG,CAAC;QAC3J;KACD;IAED,IAAII,iBAAiB,MAAM,GAAG,GAAG;QAC/BG,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SACE;QACJ;QAEAA,OAAO,IAAI,CAAC;YACV,MAAM;YACN,SAASH,iBAAiB,OAAO,CAAC,CAACM,QAAU;oBAC3C;wBACE,MAAM;wBACN,MAAM,CAAC,4BAA4B,EAAEA,MAAM,UAAU,GAAG,EAAE,EAAE,EAAEA,MAAM,YAAY,EAAE;oBACpF;oBACA;wBACE,MAAM;wBACN,KAAKA,MAAM,OAAO;oBACpB;iBACD;QACH;IACF;IAEA,OAAOH;AACT;AAEA,SAASI,uBACPC,MAA6B,EAC7BC,OAA8B;IAE9B,MAAMC,mBAAmBF,OAAO,MAAM,CACpC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,WACJH,QAAQ,cAAc,EAAE,MAAM,KAAKI,YACnCH,iBAAiB,IAAI,CAAC,CAACC,QAAUA,MAAM,GAAG,GAAG,OAC7C;IACF,MAAMG,gBACJL,QAAQ,cAAc,EAAE,mBACxBD,OAAO,IAAI,CAAC,CAACG,QAAUA,MAAM,QAAQ,GAAG;IAE1C,OAAO;QACL,YAAY;QACZ,UAAUC,YAAYG;QACtB,OAAOH,YAAY;QACnB,QAAQ;YACN,KAAKA;YACL,GAAIE,eAAe,QAAQ;gBAAE,eAAeA,cAAc,KAAK;YAAC,IAAI,CAAC,CAAC;YACtE,GAAIA,eAAe,SACf;gBAAE,gBAAgBA,cAAc,MAAM;YAAC,IACvC,CAAC,CAAC;QACR;IACF;AACF;AAEA,SAASE,uBAAuBC,OAAe;IAC7C,MAAMC,UAAUD,QAAQ,IAAI;IAC5B,MAAME,cAAcD,QAAQ,KAAK,CAAC;IAClC,OAAO,GAAIC,AAAAA,CAAAA,aAAa,CAAC,EAAE,IAAID,OAAM,EAAG,IAAI,GAAG,EAAE,CAAC;AACpD;AAEA,SAASE,mBAAmBC,WAAyB;IACnD,OAAOC,gBAAgBD;AACzB;AAEA,SAASE,yBACPC,KAAkC;IAElC,MAAM,EAAEC,OAAO,EAAEzB,gBAAgB,EAAE,GAAG0B,iCAAiCF;IACvE,MAAMzB,cAAc;QAClB,GAAG0B,OAAO;QACV,QAAQD,MAAM,MAAM;QACpB,mBAAmBA,MAAM,iBAAiB,IAAI;IAChD;IAEA,OAAO1B,iBAAiB;QACtBC;QACAC;QACA,UAAUwB,MAAM,QAAQ;QACxB,QAAQA,MAAM,MAAM;QACpB,aAAaG,qCAAqCH,MAAM,MAAM;IAChE;AACF;AAEO,MAAMI,2BAA2B,OACtCJ,OACAH;IAEA,IAAI;QACF,MAAMlB,SAASoB,yBAAyBC;QACxC,MAAMK,WAAW,MAAMC,yBACrB3B,QACAiB,mBAAmBC;QAGrB,IAAIQ,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOb,uBAAuBa,SAAS,OAAO;QAGhD,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOC,OAAO;QACd,MAAM,IAAID,MAAM,CAAC,uCAAuC,EAAEC,OAAO;IACnE;AACF;AAEO,MAAMC,iCAAiC,OAC5CT,OACAf,SACAY;IAEA,IAAI;QACF,MAAMlB,SAASoB,yBAAyBC;QACxC,MAAMU,eAAed,mBAAmBC;QACxC,IAAIZ,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EACnC,OAAO,MAAM0B,OAAOhC,QAAQ+B,cAAc;YACxC,QAAQ;YACR,SAASzB,QAAQ,OAAO;QAC1B;QAGF,MAAMoB,WAAW,MAAMC,yBAAyB3B,QAAQ+B;QACxD,IAAIL,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASb,uBAAuBa,SAAS,OAAO;YAChD,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOC,OAAO;QACd,MAAM,IAAID,MAAM,CAAC,uCAAuC,EAAEC,OAAO;IACnE;AACF;AAOO,MAAMI,mBAAmB,OAC9B5B,QACAC,SACAY,cAEOO,yBACL;QACE,GAAGnB,OAAO;QACV,QAAQF,uBAAuBC,QAAQC;QACvCD;IACF,GACAa;AAOG,MAAMgB,yBAAyB,OACpC7B,QACAC,SACAY,cAEOY,+BACL;QACE,GAAGxB,OAAO;QACV,QAAQF,uBAAuBC,QAAQC;QACvCD;IACF,GACAC,SACAY"}