@aiscene/core 9.0.6 → 9.0.8

package/dist/es/ai-model/workflows/recorder-generation/common.mjs

@@ -0,0 +1,171 @@
+import { DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS, createMidsceneRecorderMarkdownScreenshotAssets, getMidsceneRecorderEventDescription, getMidsceneRecorderSemantic } from "@aiscene/shared/recorder";
+const MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH = 1200;
+const MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH = 400;
+function cleanRecorderSemanticField(value) {
+    return value?.trim() === 'AI is analyzing element...' ? void 0 : value;
+}
+function truncateRecorderGenerationText(value, maxLength) {
+    if (value.length <= maxLength) return value;
+    return `${value.slice(0, maxLength)}... [truncated ${value.length - maxLength} chars]`;
+}
+function compactRecorderSemanticText(value) {
+    const cleaned = cleanRecorderSemanticField(value);
+    return cleaned ? truncateRecorderGenerationText(cleaned, MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH) : void 0;
+}
+function compactRecorderSemanticError(value) {
+    return value ? truncateRecorderGenerationText(value, MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH) : void 0;
+}
+function compactRecorderSemanticForGeneration(semantic) {
+    if (!semantic) return;
+    return {
+        source: semantic.source,
+        status: semantic.status,
+        confidence: semantic.confidence,
+        elementDescription: compactRecorderSemanticText(semantic.elementDescription),
+        replayInstruction: compactRecorderSemanticText(semantic.replayInstruction),
+        actionSummary: compactRecorderSemanticText(semantic.actionSummary),
+        error: compactRecorderSemanticError(semantic.error),
+        ...semantic.aiDescribe ? {
+            aiDescribe: {
+                verifyPrompt: semantic.aiDescribe.verifyPrompt,
+                verifyPassed: semantic.aiDescribe.verifyPassed,
+                deepLocate: semantic.aiDescribe.deepLocate,
+                centerDistance: semantic.aiDescribe.centerDistance,
+                expectedCenter: semantic.aiDescribe.expectedCenter,
+                actualCenter: semantic.aiDescribe.actualCenter
+            }
+        } : {}
+    };
+}
+const validateEvents = (events)=>{
+    if (!events.length) throw new Error('No events provided for test generation');
+};
+const getScreenshotsForLLM = (events, maxScreenshots = 1)=>createMidsceneRecorderMarkdownScreenshotAssets(events, {
+        baseDir: './screenshots',
+        maxScreenshots
+    }).map((asset)=>asset.dataUrl);
+const filterEventsByType = (events)=>({
+        navigationEvents: events.filter((event)=>'navigation' === event.type),
+        clickEvents: events.filter((event)=>'click' === event.type),
+        inputEvents: events.filter((event)=>'input' === event.type),
+        scrollEvents: events.filter((event)=>'scroll' === event.type)
+    });
+const createEventCounts = (filteredEvents, totalEvents)=>({
+        navigation: filteredEvents.navigationEvents.length,
+        click: filteredEvents.clickEvents.length,
+        input: filteredEvents.inputEvents.length,
+        scroll: filteredEvents.scrollEvents.length,
+        total: totalEvents
+    });
+const extractInputDescriptions = (inputEvents)=>inputEvents.map((event)=>{
+        const semantic = getMidsceneRecorderSemantic(event);
+        return {
+            description: cleanRecorderSemanticField(semantic?.elementDescription) || '',
+            value: event.value || ''
+        };
+    }).filter((item)=>item.description && item.value);
+const processEventsForLLM = (events, screenshotPathByEventHash = new Map())=>{
+    let inputIndex = 0;
+    return events.map((event, index)=>{
+        const previousEvent = events[index - 1];
+        const nextEvent = events[index + 1];
+        const previousInput = events.slice(0, index).reverse().find((candidate)=>'input' === candidate.type);
+        const nextInput = events.slice(index + 1).find((candidate)=>'input' === candidate.type);
+        const isInput = 'input' === event.type;
+        const inputSequenceIndex = isInput ? ++inputIndex : void 0;
+        const hasNeighborInput = Boolean(previousInput || nextInput);
+        const neighborInputValues = isInput ? [
+            previousInput?.value,
+            nextInput?.value
+        ].filter((value)=>Boolean(value)) : void 0;
+        const semantic = compactRecorderSemanticForGeneration(getMidsceneRecorderSemantic(event));
+        return {
+            hashId: event.hashId,
+            type: event.type,
+            timestamp: event.timestamp,
+            source: event.source,
+            actionType: event.actionType,
+            url: event.url,
+            title: event.title,
+            semantic,
+            description: getMidsceneRecorderEventDescription(event),
+            value: event.value,
+            previousActionDescription: previousEvent ? getMidsceneRecorderEventDescription(previousEvent) : void 0,
+            nextActionDescription: nextEvent ? getMidsceneRecorderEventDescription(nextEvent) : void 0,
+            ...isInput ? {
+                typedText: event.value || '',
+                inputIndex: inputSequenceIndex,
+                isSequentialInput: previousEvent?.type === 'input' || nextEvent?.type === 'input',
+                hasNeighborInput,
+                previousInputDescription: previousInput ? getMidsceneRecorderEventDescription(previousInput) : void 0,
+                neighborInputValues: neighborInputValues && neighborInputValues.length > 0 ? neighborInputValues : void 0
+            } : {},
+            pageInfo: event.pageInfo,
+            elementRect: event.elementRect,
+            screenshotPath: screenshotPathByEventHash.get(event.hashId)
+        };
+    });
+};
+const prepareEventSummary = (events, options = {})=>{
+    const filteredEvents = filterEventsByType(events);
+    const eventCounts = createEventCounts(filteredEvents, events.length);
+    const startUrl = filteredEvents.navigationEvents.length > 0 ? filteredEvents.navigationEvents[0].url || '' : '';
+    const clickDescriptions = filteredEvents.clickEvents.map((event)=>getMidsceneRecorderSemantic(event)?.elementDescription).filter((desc)=>Boolean(desc)).slice(0, 10);
+    const inputDescriptions = extractInputDescriptions(filteredEvents.inputEvents).slice(0, 10);
+    const urls = filteredEvents.navigationEvents.map((e)=>e.url).filter((url)=>Boolean(url)).slice(0, 5);
+    const processedEvents = processEventsForLLM(events, options.screenshotPathByEventHash);
+    return {
+        testName: options.testName || 'Automated test from recorded events',
+        startUrl,
+        eventCounts,
+        urls,
+        clickDescriptions,
+        inputDescriptions,
+        events: processedEvents
+    };
+};
+function prepareRecorderGenerationContext(input) {
+    validateEvents(input.events);
+    const maxScreenshots = input.maxScreenshots ?? DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS;
+    const screenshotAssets = createMidsceneRecorderMarkdownScreenshotAssets(input.events, {
+        baseDir: './screenshots',
+        maxScreenshots
+    });
+    const screenshotPathByEventHash = new Map(screenshotAssets.map((asset)=>[
+            asset.eventHashId,
+            asset.relativePath
+        ]));
+    return {
+        summary: prepareEventSummary(input.events, {
+            testName: input.testName,
+            screenshotPathByEventHash
+        }),
+        screenshotAssets
+    };
+}
+const createMessageContent = (promptText, screenshots = [], includeScreenshots = true)=>{
+    const messageContent = [
+        {
+            type: 'text',
+            text: promptText
+        }
+    ];
+    if (includeScreenshots && screenshots.length > 0) {
+        messageContent.unshift({
+            type: 'text',
+            text: 'Here are screenshots from the recording session to help you understand the context:'
+        });
+        screenshots.forEach((screenshot)=>{
+            messageContent.push({
+                type: 'image_url',
+                image_url: {
+                    url: screenshot
+                }
+            });
+        });
+    }
+    return messageContent;
+};
+export { compactRecorderSemanticForGeneration, createEventCounts, createMessageContent, extractInputDescriptions, filterEventsByType, getScreenshotsForLLM, prepareEventSummary, prepareRecorderGenerationContext, processEventsForLLM, validateEvents };
+
+//# sourceMappingURL=common.mjs.map

package/dist/es/ai-model/workflows/recorder-generation/common.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/recorder-generation/common.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/common.ts"],"sourcesContent":["import {\n  DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS,\n  type MidsceneRecorderEvent,\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  type MidsceneRecorderSemantic,\n  type MidsceneRecorderTarget,\n  createMidsceneRecorderMarkdownScreenshotAssets,\n  getMidsceneRecorderEventDescription,\n  getMidsceneRecorderSemantic,\n} from '@aiscene/shared/recorder';\n\nexport interface EventCounts {\n  navigation: number;\n  click: number;\n  input: number;\n  scroll: number;\n  total: number;\n}\n\nexport interface InputDescription {\n  description: string;\n  value: string;\n}\n\nexport interface ProcessedEvent {\n  hashId: string;\n  type: string;\n  timestamp: number;\n  source?: string;\n  actionType?: string;\n  url?: string;\n  title?: string;\n  semantic?: MidsceneRecorderSemantic;\n  description?: string;\n  value?: string;\n  typedText?: string;\n  inputIndex?: number;\n  isSequentialInput?: boolean;\n  hasNeighborInput?: boolean;\n  previousInputDescription?: string;\n  previousActionDescription?: string;\n  nextActionDescription?: string;\n  neighborInputValues?: string[];\n  pageInfo?: any;\n  elementRect?: any;\n  screenshotPath?: string;\n}\n\nexport interface EventSummary {\n  testName: string;\n  startUrl: string;\n  eventCounts: EventCounts;\n  urls: string[];\n  clickDescriptions: string[];\n  inputDescriptions: InputDescription[];\n  events: ProcessedEvent[];\n}\n\nexport interface RecorderGenerationContext {\n  summary: EventSummary;\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n}\n\nexport type ChromeRecordedEvent = MidsceneRecorderEvent;\n\nconst MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH = 1200;\nconst MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH = 400;\n\nexport interface RecorderGenerationOptions {\n  testName?: string;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  /** Language for human-readable generated content (e.g. 'English', 'Chinese'). Keys and API names are kept as-is. */\n  language?: string;\n  navigationInfo?: {\n    urls?: string[];\n    titles?: string[];\n    initialViewport?: {\n      width?: number;\n      height?: number;\n    };\n  };\n}\n\nexport interface RecorderGenerationInput extends RecorderGenerationOptions {\n  target: MidsceneRecorderTarget;\n  events: MidsceneRecorderEvent[];\n}\n\nexport interface FilteredEvents {\n  navigationEvents: ChromeRecordedEvent[];\n  clickEvents: ChromeRecordedEvent[];\n  inputEvents: ChromeRecordedEvent[];\n  scrollEvents: ChromeRecordedEvent[];\n}\n\nfunction cleanRecorderSemanticField(value?: string) {\n  return value?.trim() === 'AI is analyzing element...' ? undefined : value;\n}\n\nfunction truncateRecorderGenerationText(value: string, maxLength: number) {\n  if (value.length <= maxLength) {\n    return value;\n  }\n  return `${value.slice(0, maxLength)}... [truncated ${value.length - maxLength} chars]`;\n}\n\nfunction compactRecorderSemanticText(value?: string) {\n  const cleaned = cleanRecorderSemanticField(value);\n  return cleaned\n    ? truncateRecorderGenerationText(\n        cleaned,\n        MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH,\n      )\n    : undefined;\n}\n\nfunction compactRecorderSemanticError(value?: string) {\n  return value\n    ? truncateRecorderGenerationText(\n        value,\n        MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH,\n      )\n    : undefined;\n}\n\nexport function compactRecorderSemanticForGeneration(\n  semantic?: MidsceneRecorderSemantic,\n): MidsceneRecorderSemantic | undefined {\n  if (!semantic) {\n    return undefined;\n  }\n\n  return {\n    source: semantic.source,\n    status: semantic.status,\n    confidence: semantic.confidence,\n    elementDescription: compactRecorderSemanticText(\n      semantic.elementDescription,\n    ),\n    replayInstruction: compactRecorderSemanticText(semantic.replayInstruction),\n    actionSummary: compactRecorderSemanticText(semantic.actionSummary),\n    error: compactRecorderSemanticError(semantic.error),\n    ...(semantic.aiDescribe\n      ? {\n          aiDescribe: {\n            verifyPrompt: semantic.aiDescribe.verifyPrompt,\n            verifyPassed: semantic.aiDescribe.verifyPassed,\n            deepLocate: semantic.aiDescribe.deepLocate,\n            centerDistance: semantic.aiDescribe.centerDistance,\n            expectedCenter: semantic.aiDescribe.expectedCenter,\n            actualCenter: semantic.aiDescribe.actualCenter,\n          },\n        }\n      : {}),\n  };\n}\n\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n  if (!events.length) {\n    throw new Error('No events provided for test generation');\n  }\n};\n\nexport const getScreenshotsForLLM = (\n  events: ChromeRecordedEvent[],\n  maxScreenshots = 1,\n): string[] => {\n  return createMidsceneRecorderMarkdownScreenshotAssets(events, {\n    baseDir: './screenshots',\n    maxScreenshots,\n  }).map((asset) => asset.dataUrl);\n};\n\nexport const filterEventsByType = (\n  events: ChromeRecordedEvent[],\n): FilteredEvents => {\n  return {\n    navigationEvents: events.filter((event) => event.type === 'navigation'),\n    clickEvents: events.filter((event) => event.type === 'click'),\n    inputEvents: events.filter((event) => event.type === 'input'),\n    scrollEvents: events.filter((event) => event.type === 'scroll'),\n  };\n};\n\nexport const createEventCounts = (\n  filteredEvents: FilteredEvents,\n  totalEvents: number,\n): EventCounts => {\n  return {\n    navigation: filteredEvents.navigationEvents.length,\n    click: filteredEvents.clickEvents.length,\n    input: filteredEvents.inputEvents.length,\n    scroll: filteredEvents.scrollEvents.length,\n    total: totalEvents,\n  };\n};\n\nexport const extractInputDescriptions = (\n  inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n  return inputEvents\n    .map((event) => {\n      const semantic = getMidsceneRecorderSemantic(event);\n      return {\n        description:\n          cleanRecorderSemanticField(semantic?.elementDescription) || '',\n        value: event.value || '',\n      };\n    })\n    .filter((item) => item.description && item.value);\n};\n\nexport const processEventsForLLM = (\n  events: ChromeRecordedEvent[],\n  screenshotPathByEventHash: Map<string, string> = new Map(),\n): ProcessedEvent[] => {\n  let inputIndex = 0;\n  return events.map((event, index) => {\n    const previousEvent = events[index - 1];\n    const nextEvent = events[index + 1];\n    const previousInput = events\n      .slice(0, index)\n      .reverse()\n      .find((candidate) => candidate.type === 'input');\n    const nextInput = events\n      .slice(index + 1)\n      .find((candidate) => candidate.type === 'input');\n    const isInput = event.type === 'input';\n    const inputSequenceIndex = isInput ? ++inputIndex : undefined;\n    const hasNeighborInput = Boolean(previousInput || nextInput);\n    const neighborInputValues = isInput\n      ? [previousInput?.value, nextInput?.value].filter(\n          (value): value is string => Boolean(value),\n        )\n      : undefined;\n    const semantic = compactRecorderSemanticForGeneration(\n      getMidsceneRecorderSemantic(event),\n    );\n\n    return {\n      hashId: event.hashId,\n      type: event.type,\n      timestamp: event.timestamp,\n      source: event.source,\n      actionType: event.actionType,\n      url: event.url,\n      title: event.title,\n      semantic,\n      description: getMidsceneRecorderEventDescription(event),\n      value: event.value,\n      previousActionDescription: previousEvent\n        ? getMidsceneRecorderEventDescription(previousEvent)\n        : undefined,\n      nextActionDescription: nextEvent\n        ? getMidsceneRecorderEventDescription(nextEvent)\n        : undefined,\n      ...(isInput\n        ? {\n            typedText: event.value || '',\n            inputIndex: inputSequenceIndex,\n            isSequentialInput:\n              previousEvent?.type === 'input' || nextEvent?.type === 'input',\n            hasNeighborInput,\n            previousInputDescription: previousInput\n              ? getMidsceneRecorderEventDescription(previousInput)\n              : undefined,\n            neighborInputValues:\n              neighborInputValues && neighborInputValues.length > 0\n                ? neighborInputValues\n                : undefined,\n          }\n        : {}),\n      pageInfo: event.pageInfo,\n      elementRect: event.elementRect,\n      screenshotPath: screenshotPathByEventHash.get(event.hashId),\n    };\n  });\n};\n\nexport const prepareEventSummary = (\n  events: ChromeRecordedEvent[],\n  options: {\n    testName?: string;\n    maxScreenshots?: number;\n    screenshotPathByEventHash?: Map<string, string>;\n  } = {},\n): EventSummary => {\n  const filteredEvents = filterEventsByType(events);\n  const eventCounts = createEventCounts(filteredEvents, events.length);\n\n  const startUrl =\n    filteredEvents.navigationEvents.length > 0\n      ? filteredEvents.navigationEvents[0].url || ''\n      : '';\n\n  const clickDescriptions = filteredEvents.clickEvents\n    .map((event) => getMidsceneRecorderSemantic(event)?.elementDescription)\n    .filter((desc): desc is string => Boolean(desc))\n    .slice(0, 10);\n\n  const inputDescriptions = extractInputDescriptions(\n    filteredEvents.inputEvents,\n  ).slice(0, 10);\n\n  const urls = filteredEvents.navigationEvents\n    .map((e) => e.url)\n    .filter((url): url is string => Boolean(url))\n    .slice(0, 5);\n\n  const processedEvents = processEventsForLLM(\n    events,\n    options.screenshotPathByEventHash,\n  );\n\n  return {\n    testName: options.testName || 'Automated test from recorded events',\n    startUrl,\n    eventCounts,\n    urls,\n    clickDescriptions,\n    inputDescriptions,\n    events: processedEvents,\n  };\n};\n\nexport function prepareRecorderGenerationContext(\n  input: RecorderGenerationInput,\n): RecorderGenerationContext {\n  validateEvents(input.events);\n\n  const maxScreenshots =\n    input.maxScreenshots ?? DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS;\n  const screenshotAssets = createMidsceneRecorderMarkdownScreenshotAssets(\n    input.events,\n    {\n      baseDir: './screenshots',\n      maxScreenshots,\n    },\n  );\n  const screenshotPathByEventHash = new Map(\n    screenshotAssets.map((asset) => [asset.eventHashId, asset.relativePath]),\n  );\n\n  return {\n    summary: prepareEventSummary(input.events, {\n      testName: input.testName,\n      screenshotPathByEventHash,\n    }),\n    screenshotAssets,\n  };\n}\n\nexport const createMessageContent = (\n  promptText: string,\n  screenshots: string[] = [],\n  includeScreenshots = true,\n) => {\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  if (includeScreenshots && screenshots.length > 0) {\n    messageContent.unshift({\n      type: 'text',\n      text: 'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    screenshots.forEach((screenshot) => {\n      messageContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      });\n    });\n  }\n\n  return messageContent;\n};\n"],"names":["MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH","MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH","cleanRecorderSemanticField","value","undefined","truncateRecorderGenerationText","maxLength","compactRecorderSemanticText","cleaned","compactRecorderSemanticError","compactRecorderSemanticForGeneration","semantic","validateEvents","events","Error","getScreenshotsForLLM","maxScreenshots","createMidsceneRecorderMarkdownScreenshotAssets","asset","filterEventsByType","event","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","getMidsceneRecorderSemantic","item","processEventsForLLM","screenshotPathByEventHash","Map","inputIndex","index","previousEvent","nextEvent","previousInput","candidate","nextInput","isInput","inputSequenceIndex","hasNeighborInput","Boolean","neighborInputValues","getMidsceneRecorderEventDescription","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","inputDescriptions","urls","e","url","processedEvents","prepareRecorderGenerationContext","input","DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS","screenshotAssets","createMessageContent","promptText","screenshots","includeScreenshots","messageContent","screenshot"],"mappings":";AAiEA,MAAMA,+CAA+C;AACrD,MAAMC,gDAAgD;AA+BtD,SAASC,2BAA2BC,KAAc;IAChD,OAAOA,OAAO,WAAW,+BAA+BC,SAAYD;AACtE;AAEA,SAASE,+BAA+BF,KAAa,EAAEG,SAAiB;IACtE,IAAIH,MAAM,MAAM,IAAIG,WAClB,OAAOH;IAET,OAAO,GAAGA,MAAM,KAAK,CAAC,GAAGG,WAAW,eAAe,EAAEH,MAAM,MAAM,GAAGG,UAAU,OAAO,CAAC;AACxF;AAEA,SAASC,4BAA4BJ,KAAc;IACjD,MAAMK,UAAUN,2BAA2BC;IAC3C,OAAOK,UACHH,+BACEG,SACAR,gDAEFI;AACN;AAEA,SAASK,6BAA6BN,KAAc;IAClD,OAAOA,QACHE,+BACEF,OACAF,iDAEFG;AACN;AAEO,SAASM,qCACdC,QAAmC;IAEnC,IAAI,CAACA,UACH;IAGF,OAAO;QACL,QAAQA,SAAS,MAAM;QACvB,QAAQA,SAAS,MAAM;QACvB,YAAYA,SAAS,UAAU;QAC/B,oBAAoBJ,4BAClBI,SAAS,kBAAkB;QAE7B,mBAAmBJ,4BAA4BI,SAAS,iBAAiB;QACzE,eAAeJ,4BAA4BI,SAAS,aAAa;QACjE,OAAOF,6BAA6BE,SAAS,KAAK;QAClD,GAAIA,SAAS,UAAU,GACnB;YACE,YAAY;gBACV,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,YAAYA,SAAS,UAAU,CAAC,UAAU;gBAC1C,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,cAAcA,SAAS,UAAU,CAAC,YAAY;YAChD;QACF,IACA,CAAC,CAAC;IACR;AACF;AAEO,MAAMC,iBAAiB,CAACC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIC,MAAM;AAEpB;AAEO,MAAMC,uBAAuB,CAClCF,QACAG,iBAAiB,CAAC,GAEXC,+CAA+CJ,QAAQ;QAC5D,SAAS;QACTG;IACF,GAAG,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO;AAG1B,MAAMC,qBAAqB,CAChCN,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAGK,MAAMC,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAGK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACL;QACJ,MAAMT,WAAWe,4BAA4BN;QAC7C,OAAO;YACL,aACElB,2BAA2BS,UAAU,uBAAuB;YAC9D,OAAOS,MAAM,KAAK,IAAI;QACxB;IACF,GACC,MAAM,CAAC,CAACO,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAG7C,MAAMC,sBAAsB,CACjCf,QACAgB,4BAAiD,IAAIC,KAAK;IAE1D,IAAIC,aAAa;IACjB,OAAOlB,OAAO,GAAG,CAAC,CAACO,OAAOY;QACxB,MAAMC,gBAAgBpB,MAAM,CAACmB,QAAQ,EAAE;QACvC,MAAME,YAAYrB,MAAM,CAACmB,QAAQ,EAAE;QACnC,MAAMG,gBAAgBtB,OACnB,KAAK,CAAC,GAAGmB,OACT,OAAO,GACP,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAMC,YAAYxB,OACf,KAAK,CAACmB,QAAQ,GACd,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAME,UAAUlB,AAAe,YAAfA,MAAM,IAAI;QAC1B,MAAMmB,qBAAqBD,UAAU,EAAEP,aAAa3B;QACpD,MAAMoC,mBAAmBC,QAAQN,iBAAiBE;QAClD,MAAMK,sBAAsBJ,UACxB;YAACH,eAAe;YAAOE,WAAW;SAAM,CAAC,MAAM,CAC7C,CAAClC,QAA2BsC,QAAQtC,UAEtCC;QACJ,MAAMO,WAAWD,qCACfgB,4BAA4BN;QAG9B,OAAO;YACL,QAAQA,MAAM,MAAM;YACpB,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,QAAQA,MAAM,MAAM;YACpB,YAAYA,MAAM,UAAU;YAC5B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClBT;YACA,aAAagC,oCAAoCvB;YACjD,OAAOA,MAAM,KAAK;YAClB,2BAA2Ba,gBACvBU,oCAAoCV,iBACpC7B;YACJ,uBAAuB8B,YACnBS,oCAAoCT,aACpC9B;YACJ,GAAIkC,UACA;gBACE,WAAWlB,MAAM,KAAK,IAAI;gBAC1B,YAAYmB;gBACZ,mBACEN,eAAe,SAAS,WAAWC,WAAW,SAAS;gBACzDM;gBACA,0BAA0BL,gBACtBQ,oCAAoCR,iBACpC/B;gBACJ,qBACEsC,uBAAuBA,oBAAoB,MAAM,GAAG,IAChDA,sBACAtC;YACR,IACA,CAAC,CAAC;YACN,UAAUgB,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;YAC9B,gBAAgBS,0BAA0B,GAAG,CAACT,MAAM,MAAM;QAC5D;IACF;AACF;AAEO,MAAMwB,sBAAsB,CACjC/B,QACAgC,UAII,CAAC,CAAC;IAEN,MAAMvB,iBAAiBH,mBAAmBN;IAC1C,MAAMiC,cAAczB,kBAAkBC,gBAAgBT,OAAO,MAAM;IAEnE,MAAMkC,WACJzB,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAM0B,oBAAoB1B,eAAe,WAAW,CACjD,GAAG,CAAC,CAACF,QAAUM,4BAA4BN,QAAQ,oBACnD,MAAM,CAAC,CAAC6B,OAAyBR,QAAQQ,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAMC,oBAAoB1B,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAM6B,OAAO7B,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAAC8B,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBZ,QAAQY,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkB1B,oBACtBf,QACAgC,QAAQ,yBAAyB;IAGnC,OAAO;QACL,UAAUA,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAK;QACAH;QACAE;QACA,QAAQI;IACV;AACF;AAEO,SAASC,iCACdC,KAA8B;IAE9B5C,eAAe4C,MAAM,MAAM;IAE3B,MAAMxC,iBACJwC,MAAM,cAAc,IAAIC;IAC1B,MAAMC,mBAAmBzC,+CACvBuC,MAAM,MAAM,EACZ;QACE,SAAS;QACTxC;IACF;IAEF,MAAMa,4BAA4B,IAAIC,IACpC4B,iBAAiB,GAAG,CAAC,CAACxC,QAAU;YAACA,MAAM,WAAW;YAAEA,MAAM,YAAY;SAAC;IAGzE,OAAO;QACL,SAAS0B,oBAAoBY,MAAM,MAAM,EAAE;YACzC,UAAUA,MAAM,QAAQ;YACxB3B;QACF;QACA6B;IACF;AACF;AAEO,MAAMC,uBAAuB,CAClCC,YACAC,cAAwB,EAAE,EAC1BC,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMH;QACR;KACD;IAED,IAAIE,sBAAsBD,YAAY,MAAM,GAAG,GAAG;QAChDE,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAF,YAAY,OAAO,CAAC,CAACG;YACnBD,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKC;gBACP;YACF;QACF;IACF;IAEA,OAAOD;AACT"}

package/dist/es/ai-model/workflows/recorder-generation/index.mjs

@@ -0,0 +1,5 @@
+import { generatePlaywrightTest, generatePlaywrightTestStream } from "./playwright.mjs";
+import { convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, generateRecorderMarkdownReplay } from "./markdown.mjs";
+import { generateRecorderSessionMetadata } from "./metadata.mjs";
+import { generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream } from "./yaml.mjs";
+export { convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, generatePlaywrightTest, generatePlaywrightTestStream, generateRecorderMarkdownReplay, generateRecorderSessionMetadata, generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream };

package/dist/es/ai-model/workflows/recorder-generation/markdown.mjs

@@ -0,0 +1,253 @@
+import { imageInfoOfBase64, parseBase64, resizeBase64ImageToJpeg } from "@aiscene/shared/img";
+import { getDebug } from "@aiscene/shared/logger";
+import { getMidsceneRecorderEventDescription, stringifyMidsceneRecorderTargetBlock } from "@aiscene/shared/recorder";
+import { getModelRuntime } from "../../models/index.mjs";
+import { callAIWithStringResponse } from "../../service-caller/index.mjs";
+import { prepareRecorderGenerationContext, validateEvents } from "./common.mjs";
+const MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET = 600000;
+const MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE = 768;
+const debugMarkdownReplay = getDebug('ai:recorder-markdown', {
+    console: true
+});
+function limitScreenshotAssetsForMarkdownReplay(screenshotAssets) {
+    let usedPayload = 0;
+    return screenshotAssets.filter((asset)=>{
+        const payloadSize = asset.dataUrl.length;
+        if (payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET || usedPayload + payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET) return false;
+        usedPayload += payloadSize;
+        return true;
+    });
+}
+async function compressScreenshotAssetForMarkdownReplay(asset) {
+    const { width, height } = await imageInfoOfBase64(asset.dataUrl);
+    const longestEdge = Math.max(width, height);
+    if (longestEdge <= MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE) return asset;
+    const scale = MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE / longestEdge;
+    const dataUrl = await resizeBase64ImageToJpeg(asset.dataUrl, {
+        sourceSize: {
+            width,
+            height
+        },
+        targetSize: {
+            width: Math.max(1, Math.round(width * scale)),
+            height: Math.max(1, Math.round(height * scale))
+        }
+    });
+    const { body, mimeType } = parseBase64(dataUrl);
+    return {
+        ...asset,
+        dataUrl,
+        base64Data: body,
+        mimeType
+    };
+}
+async function prepareScreenshotAssetsForMarkdownReplay(screenshotAssets) {
+    const compressedAssets = [];
+    for (const asset of screenshotAssets)try {
+        compressedAssets.push(await compressScreenshotAssetForMarkdownReplay(asset));
+    } catch (error) {
+        debugMarkdownReplay('failed to compress screenshot asset; keeping original %o', {
+            eventHashId: asset.eventHashId,
+            eventIndex: asset.eventIndex,
+            error
+        });
+        compressedAssets.push(asset);
+    }
+    return limitScreenshotAssetsForMarkdownReplay(compressedAssets);
+}
+function summarizeScreenshotAssets(screenshotAssets) {
+    const payloadSizes = screenshotAssets.map((asset)=>asset.dataUrl.length);
+    return {
+        count: screenshotAssets.length,
+        totalPayloadChars: payloadSizes.reduce((sum, size)=>sum + size, 0),
+        maxPayloadChars: payloadSizes.length ? Math.max(...payloadSizes) : 0
+    };
+}
+function getPromptShape(prompt) {
+    let textChars = 0;
+    let imageCount = 0;
+    for (const message of prompt){
+        const content = message.content;
+        if ('string' == typeof content) {
+            textChars += content.length;
+            continue;
+        }
+        if (Array.isArray(content)) {
+            for (const part of content)if ('object' == typeof part && part && 'type' in part) {
+                if ('text' === part.type && 'text' in part) textChars += String(part.text).length;
+                if ('image_url' === part.type) imageCount += 1;
+            }
+        }
+    }
+    return {
+        textChars,
+        imageCount
+    };
+}
+function removeOmittedScreenshotPaths(summary, screenshotAssets) {
+    const includedScreenshotPaths = new Set(screenshotAssets.map((asset)=>asset.relativePath));
+    return {
+        ...summary,
+        events: summary.events.map((event)=>event.screenshotPath && !includedScreenshotPaths.has(event.screenshotPath) ? {
+                ...event,
+                screenshotPath: void 0
+            } : event)
+    };
+}
+function getMarkdownLanguageInstruction(language) {
+    const normalizedLanguage = language?.trim();
+    if (!normalizedLanguage) return '';
+    return `
+Language requirement:
+- Write all human-readable Markdown instructions in ${normalizedLanguage}.
+- Keep file paths, URLs, platform ids, API names, and quoted UI text unchanged.`;
+}
+function normalizeGeneratedMarkdown(content) {
+    const trimmed = content.trim();
+    const fencedMatch = trimmed.match(/^```(?:md|markdown)?\s*([\s\S]*?)\s*```$/i);
+    return `${(fencedMatch?.[1] ?? trimmed).trim()}\n`;
+}
+function createModelRuntime(modelConfig) {
+    return getModelRuntime(modelConfig);
+}
+function createRecorderMarkdownReplayPrompt(input) {
+    validateEvents(input.events);
+    const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } = prepareRecorderGenerationContext(input);
+    const screenshotAssets = limitScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);
+    const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);
+    return createRecorderMarkdownReplayPromptFromContext(input, summary, screenshotAssets);
+}
+async function createRecorderMarkdownReplayPromptForGeneration(input) {
+    validateEvents(input.events);
+    const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } = prepareRecorderGenerationContext(input);
+    const screenshotAssets = await prepareScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);
+    const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);
+    const prompt = createRecorderMarkdownReplayPromptFromContext(input, summary, screenshotAssets);
+    debugMarkdownReplay('markdown replay prompt shape %o', {
+        eventCount: input.events.length,
+        maxScreenshots: input.maxScreenshots,
+        rawScreenshots: summarizeScreenshotAssets(rawScreenshotAssets),
+        includedScreenshots: summarizeScreenshotAssets(screenshotAssets),
+        prompt: getPromptShape(prompt)
+    });
+    return prompt;
+}
+function createRecorderMarkdownReplayPromptFromContext(input, summary, screenshotAssets) {
+    const screenshotIndexByEventHash = new Map(screenshotAssets.map((asset, index)=>[
+            asset.eventHashId,
+            `screenshot-${index + 1}`
+        ]));
+    const events = summary.events.map((event)=>{
+        const screenshotRef = screenshotIndexByEventHash.get(event.hashId);
+        const { screenshotPath, ...eventWithoutScreenshotPath } = event;
+        return screenshotRef ? {
+            ...eventWithoutScreenshotPath,
+            screenshotRef
+        } : eventWithoutScreenshotPath;
+    });
+    const promptPayload = {
+        testName: input.testName || summary.testName,
+        target: {
+            platformId: input.target.platformId,
+            label: input.target.label,
+            values: input.target.values
+        },
+        startUrl: summary.startUrl,
+        events,
+        screenshots: screenshotAssets.map((asset, index)=>({
+                screenshotRef: `screenshot-${index + 1}`,
+                eventIndex: asset.eventIndex,
+                eventHashId: asset.eventHashId,
+                eventType: asset.eventType,
+                description: getMidsceneRecorderEventDescription(input.events[asset.eventIndex])
+            }))
+    };
+    const promptText = `Generate a Markdown replay script for Midscene Agent. It will be executed with:
+await agent.aiAct(markdownReplayPrompt)
+
+Use only the recorder data and screenshots below.
+
+Target block:
+${stringifyMidsceneRecorderTargetBlock(input.target)}
+
+Replay goal:
+- Reproduce the recorded user workflow exactly.
+- Preserve event order.
+- Preserve the user's original intent.
+- Do not invent alternative navigation paths.
+- Do not skip, merge, reorder, or add extra user actions.
+- Prefer recorded UI text, element descriptions, URLs, input values, and scroll direction.
+- For input events, enter event.typedText/event.value exactly; do not infer or correct the text from screenshots.
+- Prefer event.semantic.replayInstruction and event.semantic.elementDescription when event.semantic.source is "aiDescribe" or "recorderAI" and event.semantic.status is "ready".
+- For scroll events, preserve the recorded scroll region from event.semantic.elementDescription/replayInstruction. If the scroll happened in a specific panel, list, table, dialog body, menu, navigation area, or content pane, keep that region in the Markdown step instead of generalizing it to the whole page.
+- If event.semantic.source is "heuristic" or event.semantic.status is "pending"/"failed", use the screenshot/context to write the best visual instruction.
+- Coordinates are only fallback hints. Do not make coordinates the primary instruction when text or screenshots are available.
+- For a click/tap that only focuses a field before an input event, describe the target as the field/control itself. Do not target a placeholder character, typed character, caret, or inner text fragment inside the field.
+- If a target cannot be found, stop and report the missing step. Do not click similar-looking elements.
+- Screenshots are only generation-time visual evidence for you. The generated Markdown will be passed directly to agent.aiAct(markdownReplayPrompt), which accepts text only and cannot receive attached images.
+- Convert any useful screenshot evidence into textual replay instructions. Do not include screenshots, image syntax, image paths, or reference-image names in the generated Markdown.
+- Never write Markdown image syntax such as ![step context](...), reference-style images, HTML <img> tags, ./screenshots/... paths, or screenshot-* names in the output.
+
+Required structure:
+# ${input.testName || summary.testName}
+
+## Goal
+Reproduce the recorded user workflow exactly.
+
+## Target
+- Platform: ${input.target.platformId}
+- Start target: ${summary.startUrl || input.target.label || input.target.deviceId || 'Recorded target'}
+
+## Steps
+1. ...
+
+Recorder data:
+${JSON.stringify(promptPayload, null, 2)}${getMarkdownLanguageInstruction(input.language)}
+
+Important: Return ONLY raw Markdown. Do NOT wrap the response in markdown code blocks.`;
+    const content = [
+        {
+            type: 'text',
+            text: promptText
+        }
+    ];
+    for (const asset of screenshotAssets){
+        const screenshotRef = screenshotIndexByEventHash.get(asset.eventHashId);
+        content.push({
+            type: 'text',
+            text: `${screenshotRef} for event #${asset.eventIndex + 1}`
+        });
+        content.push({
+            type: 'image_url',
+            image_url: {
+                url: asset.dataUrl
+            }
+        });
+    }
+    return [
+        {
+            role: 'system',
+            content: "You generate precise Markdown replay scripts for Midscene agent.aiAct. The final output is plain text that will be passed directly to agent.aiAct, so it must be deterministic, ordered, safe for AI execution, and must not contain image references, screenshot paths, or screenshot labels."
+        },
+        {
+            role: 'user',
+            content
+        }
+    ];
+}
+async function generateRecorderMarkdownReplay(input, modelConfig) {
+    try {
+        const prompt = await createRecorderMarkdownReplayPromptForGeneration(input);
+        const response = await callAIWithStringResponse(prompt, createModelRuntime(modelConfig));
+        if (response?.content && 'string' == typeof response.content) return normalizeGeneratedMarkdown(response.content);
+        throw new Error('Failed to generate recorder Markdown replay');
+    } catch (error) {
+        throw new Error(`Failed to generate recorder Markdown replay: ${error}`);
+    }
+}
+async function convertRecordLogIntoMarkdown(log, modelConfig) {
+    return generateRecorderMarkdownReplay(log, modelConfig);
+}
+export { convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, generateRecorderMarkdownReplay };
+
+//# sourceMappingURL=markdown.mjs.map

package/dist/es/ai-model/workflows/recorder-generation/markdown.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/recorder-generation/markdown.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/markdown.ts"],"sourcesContent":["import type { IModelConfig } from '@aiscene/shared/env';\nimport {\n  imageInfoOfBase64,\n  parseBase64,\n  resizeBase64ImageToJpeg,\n} from '@aiscene/shared/img';\nimport { getDebug } from '@aiscene/shared/logger';\nimport {\n  type MidsceneRecorderMarkdownScreenshotAsset,\n  getMidsceneRecorderEventDescription,\n  stringifyMidsceneRecorderTargetBlock,\n} from '@aiscene/shared/recorder';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { getModelRuntime } from '../../models';\nimport { callAIWithStringResponse } from '../../service-caller';\nimport {\n  type RecorderGenerationInput,\n  prepareRecorderGenerationContext,\n  validateEvents,\n} from './common';\n\nexport type RecorderMarkdownGenerationInput = RecorderGenerationInput;\n\nconst MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET = 600_000;\nconst MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE = 768;\nconst debugMarkdownReplay = getDebug('ai:recorder-markdown', {\n  console: true,\n});\n\nfunction limitScreenshotAssetsForMarkdownReplay(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  let usedPayload = 0;\n  return screenshotAssets.filter((asset) => {\n    const payloadSize = asset.dataUrl.length;\n    if (\n      payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET ||\n      usedPayload + payloadSize > MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET\n    ) {\n      return false;\n    }\n    usedPayload += payloadSize;\n    return true;\n  });\n}\n\nasync function compressScreenshotAssetForMarkdownReplay(\n  asset: MidsceneRecorderMarkdownScreenshotAsset,\n): Promise<MidsceneRecorderMarkdownScreenshotAsset> {\n  const { width, height } = await imageInfoOfBase64(asset.dataUrl);\n  const longestEdge = Math.max(width, height);\n  if (longestEdge <= MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE) {\n    return asset;\n  }\n\n  const scale = MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE / longestEdge;\n  const dataUrl = await resizeBase64ImageToJpeg(asset.dataUrl, {\n    sourceSize: { width, height },\n    targetSize: {\n      width: Math.max(1, Math.round(width * scale)),\n      height: Math.max(1, Math.round(height * scale)),\n    },\n  });\n  const { body, mimeType } = parseBase64(dataUrl);\n  return {\n    ...asset,\n    dataUrl,\n    base64Data: body,\n    mimeType,\n  };\n}\n\nasync function prepareScreenshotAssetsForMarkdownReplay(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const compressedAssets: MidsceneRecorderMarkdownScreenshotAsset[] = [];\n  for (const asset of screenshotAssets) {\n    try {\n      compressedAssets.push(\n        await compressScreenshotAssetForMarkdownReplay(asset),\n      );\n    } catch (error) {\n      debugMarkdownReplay(\n        'failed to compress screenshot asset; keeping original %o',\n        {\n          eventHashId: asset.eventHashId,\n          eventIndex: asset.eventIndex,\n          error,\n        },\n      );\n      compressedAssets.push(asset);\n    }\n  }\n  return limitScreenshotAssetsForMarkdownReplay(compressedAssets);\n}\n\nfunction summarizeScreenshotAssets(\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const payloadSizes = screenshotAssets.map((asset) => asset.dataUrl.length);\n  return {\n    count: screenshotAssets.length,\n    totalPayloadChars: payloadSizes.reduce((sum, size) => sum + size, 0),\n    maxPayloadChars: payloadSizes.length ? Math.max(...payloadSizes) : 0,\n  };\n}\n\nfunction getPromptShape(prompt: ChatCompletionMessageParam[]) {\n  let textChars = 0;\n  let imageCount = 0;\n  for (const message of prompt) {\n    const content = message.content;\n    if (typeof content === 'string') {\n      textChars += content.length;\n      continue;\n    }\n    if (!Array.isArray(content)) {\n      continue;\n    }\n    for (const part of content) {\n      if (typeof part === 'object' && part && 'type' in part) {\n        if (part.type === 'text' && 'text' in part) {\n          textChars += String(part.text).length;\n        }\n        if (part.type === 'image_url') {\n          imageCount += 1;\n        }\n      }\n    }\n  }\n  return { textChars, imageCount };\n}\n\nfunction removeOmittedScreenshotPaths(\n  summary: ReturnType<typeof prepareRecorderGenerationContext>['summary'],\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n) {\n  const includedScreenshotPaths = new Set(\n    screenshotAssets.map((asset) => asset.relativePath),\n  );\n  return {\n    ...summary,\n    events: summary.events.map((event) =>\n      event.screenshotPath && !includedScreenshotPaths.has(event.screenshotPath)\n        ? { ...event, screenshotPath: undefined }\n        : event,\n    ),\n  };\n}\n\nfunction getMarkdownLanguageInstruction(language?: string) {\n  const normalizedLanguage = language?.trim();\n  if (!normalizedLanguage) {\n    return '';\n  }\n\n  return `\nLanguage requirement:\n- Write all human-readable Markdown instructions in ${normalizedLanguage}.\n- Keep file paths, URLs, platform ids, API names, and quoted UI text unchanged.`;\n}\n\nfunction normalizeGeneratedMarkdown(content: string) {\n  const trimmed = content.trim();\n  const fencedMatch = trimmed.match(\n    /^```(?:md|markdown)?\\s*([\\s\\S]*?)\\s*```$/i,\n  );\n  return `${(fencedMatch?.[1] ?? trimmed).trim()}\\n`;\n}\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\nexport function createRecorderMarkdownReplayPrompt(\n  input: RecorderMarkdownGenerationInput,\n): ChatCompletionMessageParam[] {\n  validateEvents(input.events);\n\n  const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } =\n    prepareRecorderGenerationContext(input);\n  const screenshotAssets =\n    limitScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);\n  const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);\n  return createRecorderMarkdownReplayPromptFromContext(\n    input,\n    summary,\n    screenshotAssets,\n  );\n}\n\nasync function createRecorderMarkdownReplayPromptForGeneration(\n  input: RecorderMarkdownGenerationInput,\n): Promise<ChatCompletionMessageParam[]> {\n  validateEvents(input.events);\n\n  const { summary: rawSummary, screenshotAssets: rawScreenshotAssets } =\n    prepareRecorderGenerationContext(input);\n  const screenshotAssets =\n    await prepareScreenshotAssetsForMarkdownReplay(rawScreenshotAssets);\n  const summary = removeOmittedScreenshotPaths(rawSummary, screenshotAssets);\n  const prompt = createRecorderMarkdownReplayPromptFromContext(\n    input,\n    summary,\n    screenshotAssets,\n  );\n  debugMarkdownReplay('markdown replay prompt shape %o', {\n    eventCount: input.events.length,\n    maxScreenshots: input.maxScreenshots,\n    rawScreenshots: summarizeScreenshotAssets(rawScreenshotAssets),\n    includedScreenshots: summarizeScreenshotAssets(screenshotAssets),\n    prompt: getPromptShape(prompt),\n  });\n  return prompt;\n}\n\nfunction createRecorderMarkdownReplayPromptFromContext(\n  input: RecorderMarkdownGenerationInput,\n  summary: ReturnType<typeof prepareRecorderGenerationContext>['summary'],\n  screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[],\n): ChatCompletionMessageParam[] {\n  const screenshotIndexByEventHash = new Map(\n    screenshotAssets.map((asset, index) => [\n      asset.eventHashId,\n      `screenshot-${index + 1}`,\n    ]),\n  );\n  const events = summary.events.map((event) => {\n    const screenshotRef = screenshotIndexByEventHash.get(event.hashId);\n    const { screenshotPath, ...eventWithoutScreenshotPath } = event;\n    return screenshotRef\n      ? { ...eventWithoutScreenshotPath, screenshotRef }\n      : eventWithoutScreenshotPath;\n  });\n  const promptPayload = {\n    testName: input.testName || summary.testName,\n    target: {\n      platformId: input.target.platformId,\n      label: input.target.label,\n      values: input.target.values,\n    },\n    startUrl: summary.startUrl,\n    events,\n    screenshots: screenshotAssets.map((asset, index) => ({\n      screenshotRef: `screenshot-${index + 1}`,\n      eventIndex: asset.eventIndex,\n      eventHashId: asset.eventHashId,\n      eventType: asset.eventType,\n      description: getMidsceneRecorderEventDescription(\n        input.events[asset.eventIndex],\n      ),\n    })),\n  };\n  const promptText = `Generate a Markdown replay script for Midscene Agent. It will be executed with:\nawait agent.aiAct(markdownReplayPrompt)\n\nUse only the recorder data and screenshots below.\n\nTarget block:\n${stringifyMidsceneRecorderTargetBlock(input.target)}\n\nReplay goal:\n- Reproduce the recorded user workflow exactly.\n- Preserve event order.\n- Preserve the user's original intent.\n- Do not invent alternative navigation paths.\n- Do not skip, merge, reorder, or add extra user actions.\n- Prefer recorded UI text, element descriptions, URLs, input values, and scroll direction.\n- For input events, enter event.typedText/event.value exactly; do not infer or correct the text from screenshots.\n- Prefer event.semantic.replayInstruction and event.semantic.elementDescription when event.semantic.source is \"aiDescribe\" or \"recorderAI\" and event.semantic.status is \"ready\".\n- For scroll events, preserve the recorded scroll region from event.semantic.elementDescription/replayInstruction. If the scroll happened in a specific panel, list, table, dialog body, menu, navigation area, or content pane, keep that region in the Markdown step instead of generalizing it to the whole page.\n- If event.semantic.source is \"heuristic\" or event.semantic.status is \"pending\"/\"failed\", use the screenshot/context to write the best visual instruction.\n- Coordinates are only fallback hints. Do not make coordinates the primary instruction when text or screenshots are available.\n- For a click/tap that only focuses a field before an input event, describe the target as the field/control itself. Do not target a placeholder character, typed character, caret, or inner text fragment inside the field.\n- If a target cannot be found, stop and report the missing step. Do not click similar-looking elements.\n- Screenshots are only generation-time visual evidence for you. The generated Markdown will be passed directly to agent.aiAct(markdownReplayPrompt), which accepts text only and cannot receive attached images.\n- Convert any useful screenshot evidence into textual replay instructions. Do not include screenshots, image syntax, image paths, or reference-image names in the generated Markdown.\n- Never write Markdown image syntax such as ![step context](...), reference-style images, HTML <img> tags, ./screenshots/... paths, or screenshot-* names in the output.\n\nRequired structure:\n# ${input.testName || summary.testName}\n\n## Goal\nReproduce the recorded user workflow exactly.\n\n## Target\n- Platform: ${input.target.platformId}\n- Start target: ${summary.startUrl || input.target.label || input.target.deviceId || 'Recorded target'}\n\n## Steps\n1. ...\n\nRecorder data:\n${JSON.stringify(promptPayload, null, 2)}${getMarkdownLanguageInstruction(input.language)}\n\nImportant: Return ONLY raw Markdown. Do NOT wrap the response in markdown code blocks.`;\n\n  const content: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  for (const asset of screenshotAssets) {\n    const screenshotRef = screenshotIndexByEventHash.get(asset.eventHashId);\n    content.push({\n      type: 'text',\n      text: `${screenshotRef} for event #${asset.eventIndex + 1}`,\n    });\n    content.push({\n      type: 'image_url',\n      image_url: {\n        url: asset.dataUrl,\n      },\n    });\n  }\n\n  return [\n    {\n      role: 'system',\n      content:\n        'You generate precise Markdown replay scripts for Midscene agent.aiAct. The final output is plain text that will be passed directly to agent.aiAct, so it must be deterministic, ordered, safe for AI execution, and must not contain image references, screenshot paths, or screenshot labels.',\n    },\n    {\n      role: 'user',\n      content,\n    },\n  ];\n}\n\nexport async function generateRecorderMarkdownReplay(\n  input: RecorderMarkdownGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> {\n  try {\n    const prompt = await createRecorderMarkdownReplayPromptForGeneration(input);\n    const response = await callAIWithStringResponse(\n      prompt,\n      createModelRuntime(modelConfig),\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return normalizeGeneratedMarkdown(response.content);\n    }\n\n    throw new Error('Failed to generate recorder Markdown replay');\n  } catch (error) {\n    throw new Error(`Failed to generate recorder Markdown replay: ${error}`);\n  }\n}\n\nexport async function convertRecordLogIntoMarkdown(\n  log: RecorderMarkdownGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<string> {\n  return generateRecorderMarkdownReplay(log, modelConfig);\n}\n"],"names":["MARKDOWN_REPLAY_SCREENSHOT_PAYLOAD_BUDGET","MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE","debugMarkdownReplay","getDebug","limitScreenshotAssetsForMarkdownReplay","screenshotAssets","usedPayload","asset","payloadSize","compressScreenshotAssetForMarkdownReplay","width","height","imageInfoOfBase64","longestEdge","Math","scale","dataUrl","resizeBase64ImageToJpeg","body","mimeType","parseBase64","prepareScreenshotAssetsForMarkdownReplay","compressedAssets","error","summarizeScreenshotAssets","payloadSizes","sum","size","getPromptShape","prompt","textChars","imageCount","message","content","Array","part","String","removeOmittedScreenshotPaths","summary","includedScreenshotPaths","Set","event","undefined","getMarkdownLanguageInstruction","language","normalizedLanguage","normalizeGeneratedMarkdown","trimmed","fencedMatch","createModelRuntime","modelConfig","getModelRuntime","createRecorderMarkdownReplayPrompt","input","validateEvents","rawSummary","rawScreenshotAssets","prepareRecorderGenerationContext","createRecorderMarkdownReplayPromptFromContext","createRecorderMarkdownReplayPromptForGeneration","screenshotIndexByEventHash","Map","index","events","screenshotRef","screenshotPath","eventWithoutScreenshotPath","promptPayload","getMidsceneRecorderEventDescription","promptText","stringifyMidsceneRecorderTargetBlock","JSON","generateRecorderMarkdownReplay","response","callAIWithStringResponse","Error","convertRecordLogIntoMarkdown","log"],"mappings":";;;;;;AAuBA,MAAMA,4CAA4C;AAClD,MAAMC,sCAAsC;AAC5C,MAAMC,sBAAsBC,SAAS,wBAAwB;IAC3D,SAAS;AACX;AAEA,SAASC,uCACPC,gBAA2D;IAE3D,IAAIC,cAAc;IAClB,OAAOD,iBAAiB,MAAM,CAAC,CAACE;QAC9B,MAAMC,cAAcD,MAAM,OAAO,CAAC,MAAM;QACxC,IACEC,cAAcR,6CACdM,cAAcE,cAAcR,2CAE5B,OAAO;QAETM,eAAeE;QACf,OAAO;IACT;AACF;AAEA,eAAeC,yCACbF,KAA8C;IAE9C,MAAM,EAAEG,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMC,kBAAkBL,MAAM,OAAO;IAC/D,MAAMM,cAAcC,KAAK,GAAG,CAACJ,OAAOC;IACpC,IAAIE,eAAeZ,qCACjB,OAAOM;IAGT,MAAMQ,QAAQd,sCAAsCY;IACpD,MAAMG,UAAU,MAAMC,wBAAwBV,MAAM,OAAO,EAAE;QAC3D,YAAY;YAAEG;YAAOC;QAAO;QAC5B,YAAY;YACV,OAAOG,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACJ,QAAQK;YACtC,QAAQD,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACH,SAASI;QAC1C;IACF;IACA,MAAM,EAAEG,IAAI,EAAEC,QAAQ,EAAE,GAAGC,YAAYJ;IACvC,OAAO;QACL,GAAGT,KAAK;QACRS;QACA,YAAYE;QACZC;IACF;AACF;AAEA,eAAeE,yCACbhB,gBAA2D;IAE3D,MAAMiB,mBAA8D,EAAE;IACtE,KAAK,MAAMf,SAASF,iBAClB,IAAI;QACFiB,iBAAiB,IAAI,CACnB,MAAMb,yCAAyCF;IAEnD,EAAE,OAAOgB,OAAO;QACdrB,oBACE,4DACA;YACE,aAAaK,MAAM,WAAW;YAC9B,YAAYA,MAAM,UAAU;YAC5BgB;QACF;QAEFD,iBAAiB,IAAI,CAACf;IACxB;IAEF,OAAOH,uCAAuCkB;AAChD;AAEA,SAASE,0BACPnB,gBAA2D;IAE3D,MAAMoB,eAAepB,iBAAiB,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO,CAAC,MAAM;IACzE,OAAO;QACL,OAAOF,iBAAiB,MAAM;QAC9B,mBAAmBoB,aAAa,MAAM,CAAC,CAACC,KAAKC,OAASD,MAAMC,MAAM;QAClE,iBAAiBF,aAAa,MAAM,GAAGX,KAAK,GAAG,IAAIW,gBAAgB;IACrE;AACF;AAEA,SAASG,eAAeC,MAAoC;IAC1D,IAAIC,YAAY;IAChB,IAAIC,aAAa;IACjB,KAAK,MAAMC,WAAWH,OAAQ;QAC5B,MAAMI,UAAUD,QAAQ,OAAO;QAC/B,IAAI,AAAmB,YAAnB,OAAOC,SAAsB;YAC/BH,aAAaG,QAAQ,MAAM;YAC3B;QACF;QACA,IAAKC,MAAM,OAAO,CAACD,UAGnB;YAAA,KAAK,MAAME,QAAQF,QACjB,IAAI,AAAgB,YAAhB,OAAOE,QAAqBA,QAAQ,UAAUA,MAAM;gBACtD,IAAIA,AAAc,WAAdA,KAAK,IAAI,IAAe,UAAUA,MACpCL,aAAaM,OAAOD,KAAK,IAAI,EAAE,MAAM;gBAEvC,IAAIA,AAAc,gBAAdA,KAAK,IAAI,EACXJ,cAAc;YAElB;QACF;IACF;IACA,OAAO;QAAED;QAAWC;IAAW;AACjC;AAEA,SAASM,6BACPC,OAAuE,EACvEjC,gBAA2D;IAE3D,MAAMkC,0BAA0B,IAAIC,IAClCnC,iBAAiB,GAAG,CAAC,CAACE,QAAUA,MAAM,YAAY;IAEpD,OAAO;QACL,GAAG+B,OAAO;QACV,QAAQA,QAAQ,MAAM,CAAC,GAAG,CAAC,CAACG,QAC1BA,MAAM,cAAc,IAAI,CAACF,wBAAwB,GAAG,CAACE,MAAM,cAAc,IACrE;gBAAE,GAAGA,KAAK;gBAAE,gBAAgBC;YAAU,IACtCD;IAER;AACF;AAEA,SAASE,+BAA+BC,QAAiB;IACvD,MAAMC,qBAAqBD,UAAU;IACrC,IAAI,CAACC,oBACH,OAAO;IAGT,OAAO,CAAC;;oDAE0C,EAAEA,mBAAmB;+EACM,CAAC;AAChF;AAEA,SAASC,2BAA2Bb,OAAe;IACjD,MAAMc,UAAUd,QAAQ,IAAI;IAC5B,MAAMe,cAAcD,QAAQ,KAAK,CAC/B;IAEF,OAAO,GAAIC,AAAAA,CAAAA,aAAa,CAAC,EAAE,IAAID,OAAM,EAAG,IAAI,GAAG,EAAE,CAAC;AACpD;AAEA,SAASE,mBAAmBC,WAAyB;IACnD,OAAOC,gBAAgBD;AACzB;AAEO,SAASE,mCACdC,KAAsC;IAEtCC,eAAeD,MAAM,MAAM;IAE3B,MAAM,EAAE,SAASE,UAAU,EAAE,kBAAkBC,mBAAmB,EAAE,GAClEC,iCAAiCJ;IACnC,MAAMhD,mBACJD,uCAAuCoD;IACzC,MAAMlB,UAAUD,6BAA6BkB,YAAYlD;IACzD,OAAOqD,8CACLL,OACAf,SACAjC;AAEJ;AAEA,eAAesD,gDACbN,KAAsC;IAEtCC,eAAeD,MAAM,MAAM;IAE3B,MAAM,EAAE,SAASE,UAAU,EAAE,kBAAkBC,mBAAmB,EAAE,GAClEC,iCAAiCJ;IACnC,MAAMhD,mBACJ,MAAMgB,yCAAyCmC;IACjD,MAAMlB,UAAUD,6BAA6BkB,YAAYlD;IACzD,MAAMwB,SAAS6B,8CACbL,OACAf,SACAjC;IAEFH,oBAAoB,mCAAmC;QACrD,YAAYmD,MAAM,MAAM,CAAC,MAAM;QAC/B,gBAAgBA,MAAM,cAAc;QACpC,gBAAgB7B,0BAA0BgC;QAC1C,qBAAqBhC,0BAA0BnB;QAC/C,QAAQuB,eAAeC;IACzB;IACA,OAAOA;AACT;AAEA,SAAS6B,8CACPL,KAAsC,EACtCf,OAAuE,EACvEjC,gBAA2D;IAE3D,MAAMuD,6BAA6B,IAAIC,IACrCxD,iBAAiB,GAAG,CAAC,CAACE,OAAOuD,QAAU;YACrCvD,MAAM,WAAW;YACjB,CAAC,WAAW,EAAEuD,QAAQ,GAAG;SAC1B;IAEH,MAAMC,SAASzB,QAAQ,MAAM,CAAC,GAAG,CAAC,CAACG;QACjC,MAAMuB,gBAAgBJ,2BAA2B,GAAG,CAACnB,MAAM,MAAM;QACjE,MAAM,EAAEwB,cAAc,EAAE,GAAGC,4BAA4B,GAAGzB;QAC1D,OAAOuB,gBACH;YAAE,GAAGE,0BAA0B;YAAEF;QAAc,IAC/CE;IACN;IACA,MAAMC,gBAAgB;QACpB,UAAUd,MAAM,QAAQ,IAAIf,QAAQ,QAAQ;QAC5C,QAAQ;YACN,YAAYe,MAAM,MAAM,CAAC,UAAU;YACnC,OAAOA,MAAM,MAAM,CAAC,KAAK;YACzB,QAAQA,MAAM,MAAM,CAAC,MAAM;QAC7B;QACA,UAAUf,QAAQ,QAAQ;QAC1ByB;QACA,aAAa1D,iBAAiB,GAAG,CAAC,CAACE,OAAOuD,QAAW;gBACnD,eAAe,CAAC,WAAW,EAAEA,QAAQ,GAAG;gBACxC,YAAYvD,MAAM,UAAU;gBAC5B,aAAaA,MAAM,WAAW;gBAC9B,WAAWA,MAAM,SAAS;gBAC1B,aAAa6D,oCACXf,MAAM,MAAM,CAAC9C,MAAM,UAAU,CAAC;YAElC;IACF;IACA,MAAM8D,aAAa,CAAC;;;;;;AAMtB,EAAEC,qCAAqCjB,MAAM,MAAM,EAAE;;;;;;;;;;;;;;;;;;;;;EAqBnD,EAAEA,MAAM,QAAQ,IAAIf,QAAQ,QAAQ,CAAC;;;;;;YAM3B,EAAEe,MAAM,MAAM,CAAC,UAAU,CAAC;gBACtB,EAAEf,QAAQ,QAAQ,IAAIe,MAAM,MAAM,CAAC,KAAK,IAAIA,MAAM,MAAM,CAAC,QAAQ,IAAI,kBAAkB;;;;;;AAMvG,EAAEkB,KAAK,SAAS,CAACJ,eAAe,MAAM,KAAKxB,+BAA+BU,MAAM,QAAQ,EAAE;;sFAEJ,CAAC;IAErF,MAAMpB,UAAiB;QACrB;YACE,MAAM;YACN,MAAMoC;QACR;KACD;IAED,KAAK,MAAM9D,SAASF,iBAAkB;QACpC,MAAM2D,gBAAgBJ,2BAA2B,GAAG,CAACrD,MAAM,WAAW;QACtE0B,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,GAAG+B,cAAc,YAAY,EAAEzD,MAAM,UAAU,GAAG,GAAG;QAC7D;QACA0B,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,WAAW;gBACT,KAAK1B,MAAM,OAAO;YACpB;QACF;IACF;IAEA,OAAO;QACL;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACN0B;QACF;KACD;AACH;AAEO,eAAeuC,+BACpBnB,KAAsC,EACtCH,WAAyB;IAEzB,IAAI;QACF,MAAMrB,SAAS,MAAM8B,gDAAgDN;QACrE,MAAMoB,WAAW,MAAMC,yBACrB7C,QACAoB,mBAAmBC;QAGrB,IAAIuB,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO3B,2BAA2B2B,SAAS,OAAO;QAGpD,MAAM,IAAIE,MAAM;IAClB,EAAE,OAAOpD,OAAO;QACd,MAAM,IAAIoD,MAAM,CAAC,6CAA6C,EAAEpD,OAAO;IACzE;AACF;AAEO,eAAeqD,6BACpBC,GAAoC,EACpC3B,WAAyB;IAEzB,OAAOsB,+BAA+BK,KAAK3B;AAC7C"}

package/dist/es/ai-model/workflows/recorder-generation/metadata.mjs

@@ -0,0 +1,83 @@
+import { getMidsceneRecorderEventDescription, getMidsceneRecorderScreenshotsForLLM, getMidsceneRecorderSemantic } from "@aiscene/shared/recorder";
+import { getModelRuntime } from "../../models/index.mjs";
+import { callAIWithObjectResponse } from "../../service-caller/index.mjs";
+import { compactRecorderSemanticForGeneration } from "./common.mjs";
+function summarizeRecorderEvents(input) {
+    const events = input.events;
+    const navigationEvents = events.filter((event)=>'navigation' === event.type);
+    const clickEvents = events.filter((event)=>'click' === event.type);
+    const inputEvents = events.filter((event)=>'input' === event.type);
+    const scrollEvents = events.filter((event)=>'scroll' === event.type);
+    const urls = navigationEvents.map((event)=>event.url).filter((url)=>Boolean(url));
+    const titles = navigationEvents.map((event)=>event.title).filter((title)=>Boolean(title));
+    return {
+        platform: input.target.platformId,
+        target: input.target,
+        fallbackName: input.fallbackName,
+        pageCount: navigationEvents.length,
+        pageTitles: titles.slice(0, 5),
+        urls: urls.slice(0, 5),
+        clickCount: clickEvents.length,
+        inputCount: inputEvents.length,
+        scrollCount: scrollEvents.length,
+        totalActions: events.length,
+        firstUrl: urls[0] || input.target.values.url || '',
+        lastUrl: urls[urls.length - 1] || '',
+        events: events.slice(0, 20).map((event)=>({
+                type: event.type,
+                actionType: event.actionType,
+                url: event.url,
+                title: event.title,
+                value: event.value,
+                description: getMidsceneRecorderEventDescription(event),
+                semantic: compactRecorderSemanticForGeneration(getMidsceneRecorderSemantic(event))
+            }))
+    };
+}
+function normalizeMetadataValue(value) {
+    return 'string' == typeof value ? value.trim() : '';
+}
+async function generateRecorderSessionMetadata(input, modelConfig) {
+    if (!input?.events?.length) throw new Error('generateRecorderSessionMetadata: events are required.');
+    if (!modelConfig?.modelName) throw new Error('generateRecorderSessionMetadata: modelConfig.modelName is required.');
+    const summary = summarizeRecorderEvents(input);
+    const screenshots = getMidsceneRecorderScreenshotsForLLM(input.events, input.maxScreenshots ?? 1);
+    const messageContent = [
+        {
+            type: 'text',
+            text: `Generate a concise title (5-7 words) and brief description (1-2 sentences) for a Studio recording of user actions.
+
+The recording can target Web, Android, iOS, HarmonyOS, or Computer. Do not assume it is a browser session unless the platform is web.
+Describe what the user did or accomplished. The description should use the user as the subject, preferably starting with "The user ...". Do not start the description with "The session ...".
+The title should be action-oriented and highlight the main task accomplished.
+
+Summary:
+${JSON.stringify(summary, null, 2)}
+
+Respond with a JSON object containing exactly "title" and "description".`
+        }
+    ];
+    for (const screenshot of screenshots)messageContent.push({
+        type: 'image_url',
+        image_url: {
+            url: screenshot
+        }
+    });
+    const response = await callAIWithObjectResponse([
+        {
+            role: 'system',
+            content: "You generate clear, task-oriented titles and descriptions for recorded automation sessions."
+        },
+        {
+            role: 'user',
+            content: messageContent
+        }
+    ], getModelRuntime(modelConfig));
+    return {
+        title: normalizeMetadataValue(response.content.title),
+        description: normalizeMetadataValue(response.content.description)
+    };
+}
+export { generateRecorderSessionMetadata };
+
+//# sourceMappingURL=metadata.mjs.map

package/dist/es/ai-model/workflows/recorder-generation/metadata.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/recorder-generation/metadata.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/metadata.ts"],"sourcesContent":["import type { IModelConfig } from '@aiscene/shared/env';\nimport {\n  type MidsceneRecorderEvent,\n  type MidsceneRecorderTarget,\n  getMidsceneRecorderEventDescription,\n  getMidsceneRecorderScreenshotsForLLM,\n  getMidsceneRecorderSemantic,\n} from '@aiscene/shared/recorder';\nimport { getModelRuntime } from '../../models';\nimport { callAIWithObjectResponse } from '../../service-caller';\nimport { compactRecorderSemanticForGeneration } from './common';\n\nexport interface RecorderMetadataGenerationInput {\n  target: MidsceneRecorderTarget;\n  events: MidsceneRecorderEvent[];\n  fallbackName?: string;\n  maxScreenshots?: number;\n}\n\nexport interface RecorderGeneratedMetadata {\n  title?: string;\n  description?: string;\n}\n\nfunction summarizeRecorderEvents(input: RecorderMetadataGenerationInput) {\n  const events = input.events;\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const clickEvents = events.filter((event) => event.type === 'click');\n  const inputEvents = events.filter((event) => event.type === 'input');\n  const scrollEvents = events.filter((event) => event.type === 'scroll');\n  const urls = navigationEvents\n    .map((event) => event.url)\n    .filter((url): url is string => Boolean(url));\n  const titles = navigationEvents\n    .map((event) => event.title)\n    .filter((title): title is string => Boolean(title));\n\n  return {\n    platform: input.target.platformId,\n    target: input.target,\n    fallbackName: input.fallbackName,\n    pageCount: navigationEvents.length,\n    pageTitles: titles.slice(0, 5),\n    urls: urls.slice(0, 5),\n    clickCount: clickEvents.length,\n    inputCount: inputEvents.length,\n    scrollCount: scrollEvents.length,\n    totalActions: events.length,\n    firstUrl: urls[0] || input.target.values.url || '',\n    lastUrl: urls[urls.length - 1] || '',\n    events: events.slice(0, 20).map((event) => ({\n      type: event.type,\n      actionType: event.actionType,\n      url: event.url,\n      title: event.title,\n      value: event.value,\n      description: getMidsceneRecorderEventDescription(event),\n      semantic: compactRecorderSemanticForGeneration(\n        getMidsceneRecorderSemantic(event),\n      ),\n    })),\n  };\n}\n\nfunction normalizeMetadataValue(value: unknown) {\n  return typeof value === 'string' ? value.trim() : '';\n}\n\nexport async function generateRecorderSessionMetadata(\n  input: RecorderMetadataGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<RecorderGeneratedMetadata> {\n  if (!input?.events?.length) {\n    throw new Error('generateRecorderSessionMetadata: events are required.');\n  }\n  if (!modelConfig?.modelName) {\n    throw new Error(\n      'generateRecorderSessionMetadata: modelConfig.modelName is required.',\n    );\n  }\n\n  const summary = summarizeRecorderEvents(input);\n  const screenshots = getMidsceneRecorderScreenshotsForLLM(\n    input.events,\n    input.maxScreenshots ?? 1,\n  );\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: `Generate a concise title (5-7 words) and brief description (1-2 sentences) for a Studio recording of user actions.\n\nThe recording can target Web, Android, iOS, HarmonyOS, or Computer. Do not assume it is a browser session unless the platform is web.\nDescribe what the user did or accomplished. The description should use the user as the subject, preferably starting with \"The user ...\". Do not start the description with \"The session ...\".\nThe title should be action-oriented and highlight the main task accomplished.\n\nSummary:\n${JSON.stringify(summary, null, 2)}\n\nRespond with a JSON object containing exactly \"title\" and \"description\".`,\n    },\n  ];\n\n  for (const screenshot of screenshots) {\n    messageContent.push({\n      type: 'image_url',\n      image_url: { url: screenshot },\n    });\n  }\n\n  const response = await callAIWithObjectResponse<{\n    title?: string;\n    description?: string;\n  }>(\n    [\n      {\n        role: 'system',\n        content:\n          'You generate clear, task-oriented titles and descriptions for recorded automation sessions.',\n      },\n      {\n        role: 'user',\n        content: messageContent,\n      },\n    ],\n    getModelRuntime(modelConfig),\n  );\n\n  return {\n    title: normalizeMetadataValue(response.content.title),\n    description: normalizeMetadataValue(response.content.description),\n  };\n}\n"],"names":["summarizeRecorderEvents","input","events","navigationEvents","event","clickEvents","inputEvents","scrollEvents","urls","url","Boolean","titles","title","getMidsceneRecorderEventDescription","compactRecorderSemanticForGeneration","getMidsceneRecorderSemantic","normalizeMetadataValue","value","generateRecorderSessionMetadata","modelConfig","Error","summary","screenshots","getMidsceneRecorderScreenshotsForLLM","messageContent","JSON","screenshot","response","callAIWithObjectResponse","getModelRuntime"],"mappings":";;;;AAwBA,SAASA,wBAAwBC,KAAsC;IACrE,MAAMC,SAASD,MAAM,MAAM;IAC3B,MAAME,mBAAmBD,OAAO,MAAM,CACpC,CAACE,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,cAAcH,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAME,cAAcJ,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAMG,eAAeL,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACxD,MAAMI,OAAOL,iBACV,GAAG,CAAC,CAACC,QAAUA,MAAM,GAAG,EACxB,MAAM,CAAC,CAACK,MAAuBC,QAAQD;IAC1C,MAAME,SAASR,iBACZ,GAAG,CAAC,CAACC,QAAUA,MAAM,KAAK,EAC1B,MAAM,CAAC,CAACQ,QAA2BF,QAAQE;IAE9C,OAAO;QACL,UAAUX,MAAM,MAAM,CAAC,UAAU;QACjC,QAAQA,MAAM,MAAM;QACpB,cAAcA,MAAM,YAAY;QAChC,WAAWE,iBAAiB,MAAM;QAClC,YAAYQ,OAAO,KAAK,CAAC,GAAG;QAC5B,MAAMH,KAAK,KAAK,CAAC,GAAG;QACpB,YAAYH,YAAY,MAAM;QAC9B,YAAYC,YAAY,MAAM;QAC9B,aAAaC,aAAa,MAAM;QAChC,cAAcL,OAAO,MAAM;QAC3B,UAAUM,IAAI,CAAC,EAAE,IAAIP,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI;QAChD,SAASO,IAAI,CAACA,KAAK,MAAM,GAAG,EAAE,IAAI;QAClC,QAAQN,OAAO,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAACE,QAAW;gBAC1C,MAAMA,MAAM,IAAI;gBAChB,YAAYA,MAAM,UAAU;gBAC5B,KAAKA,MAAM,GAAG;gBACd,OAAOA,MAAM,KAAK;gBAClB,OAAOA,MAAM,KAAK;gBAClB,aAAaS,oCAAoCT;gBACjD,UAAUU,qCACRC,4BAA4BX;YAEhC;IACF;AACF;AAEA,SAASY,uBAAuBC,KAAc;IAC5C,OAAO,AAAiB,YAAjB,OAAOA,QAAqBA,MAAM,IAAI,KAAK;AACpD;AAEO,eAAeC,gCACpBjB,KAAsC,EACtCkB,WAAyB;IAEzB,IAAI,CAAClB,OAAO,QAAQ,QAClB,MAAM,IAAImB,MAAM;IAElB,IAAI,CAACD,aAAa,WAChB,MAAM,IAAIC,MACR;IAIJ,MAAMC,UAAUrB,wBAAwBC;IACxC,MAAMqB,cAAcC,qCAClBtB,MAAM,MAAM,EACZA,MAAM,cAAc,IAAI;IAE1B,MAAMuB,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAM,CAAC;;;;;;;AAOb,EAAEC,KAAK,SAAS,CAACJ,SAAS,MAAM,GAAG;;wEAEqC,CAAC;QACrE;KACD;IAED,KAAK,MAAMK,cAAcJ,YACvBE,eAAe,IAAI,CAAC;QAClB,MAAM;QACN,WAAW;YAAE,KAAKE;QAAW;IAC/B;IAGF,MAAMC,WAAW,MAAMC,yBAIrB;QACE;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACN,SAASJ;QACX;KACD,EACDK,gBAAgBV;IAGlB,OAAO;QACL,OAAOH,uBAAuBW,SAAS,OAAO,CAAC,KAAK;QACpD,aAAaX,uBAAuBW,SAAS,OAAO,CAAC,WAAW;IAClE;AACF"}