@midscene/core 0.30.10 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent/agent.mjs +233 -144
- package/dist/es/agent/agent.mjs.map +1 -1
- package/dist/es/agent/execution-session.mjs +41 -0
- package/dist/es/agent/execution-session.mjs.map +1 -0
- package/dist/es/agent/index.mjs +3 -3
- package/dist/es/agent/task-builder.mjs +319 -0
- package/dist/es/agent/task-builder.mjs.map +1 -0
- package/dist/es/agent/task-cache.mjs +4 -4
- package/dist/es/agent/task-cache.mjs.map +1 -1
- package/dist/es/agent/tasks.mjs +197 -504
- package/dist/es/agent/tasks.mjs.map +1 -1
- package/dist/es/agent/ui-utils.mjs +54 -35
- package/dist/es/agent/ui-utils.mjs.map +1 -1
- package/dist/es/agent/utils.mjs +16 -58
- package/dist/es/agent/utils.mjs.map +1 -1
- package/dist/es/ai-model/conversation-history.mjs +25 -13
- package/dist/es/ai-model/conversation-history.mjs.map +1 -1
- package/dist/es/ai-model/index.mjs +4 -4
- package/dist/es/ai-model/inspect.mjs +45 -54
- package/dist/es/ai-model/inspect.mjs.map +1 -1
- package/dist/es/ai-model/llm-planning.mjs +47 -65
- package/dist/es/ai-model/llm-planning.mjs.map +1 -1
- package/dist/es/ai-model/prompt/assertion.mjs.map +1 -1
- package/dist/es/ai-model/prompt/common.mjs.map +1 -1
- package/dist/es/ai-model/prompt/describe.mjs.map +1 -1
- package/dist/es/ai-model/prompt/extraction.mjs.map +1 -1
- package/dist/es/ai-model/prompt/llm-locator.mjs +11 -235
- package/dist/es/ai-model/prompt/llm-locator.mjs.map +1 -1
- package/dist/es/ai-model/prompt/llm-planning.mjs +76 -322
- package/dist/es/ai-model/prompt/llm-planning.mjs.map +1 -1
- package/dist/es/ai-model/prompt/llm-section-locator.mjs +15 -14
- package/dist/es/ai-model/prompt/llm-section-locator.mjs.map +1 -1
- package/dist/es/ai-model/prompt/order-sensitive-judge.mjs +35 -0
- package/dist/es/ai-model/prompt/order-sensitive-judge.mjs.map +1 -0
- package/dist/es/ai-model/prompt/playwright-generator.mjs +2 -2
- package/dist/es/ai-model/prompt/playwright-generator.mjs.map +1 -1
- package/dist/es/ai-model/prompt/ui-tars-locator.mjs.map +1 -1
- package/dist/es/ai-model/prompt/ui-tars-planning.mjs.map +1 -1
- package/dist/es/ai-model/prompt/util.mjs +3 -88
- package/dist/es/ai-model/prompt/util.mjs.map +1 -1
- package/dist/es/ai-model/prompt/yaml-generator.mjs +10 -10
- package/dist/es/ai-model/prompt/yaml-generator.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/index.mjs +182 -274
- package/dist/es/ai-model/service-caller/index.mjs.map +1 -1
- package/dist/es/ai-model/ui-tars-planning.mjs +69 -8
- package/dist/es/ai-model/ui-tars-planning.mjs.map +1 -1
- package/dist/es/{ai-model/common.mjs → common.mjs} +18 -30
- package/dist/es/common.mjs.map +1 -0
- package/dist/es/device/device-options.mjs +0 -0
- package/dist/es/device/index.mjs +29 -12
- package/dist/es/device/index.mjs.map +1 -1
- package/dist/es/index.mjs +5 -4
- package/dist/es/index.mjs.map +1 -1
- package/dist/es/report.mjs.map +1 -1
- package/dist/es/{insight → service}/index.mjs +38 -51
- package/dist/es/service/index.mjs.map +1 -0
- package/dist/es/{insight → service}/utils.mjs +3 -3
- package/dist/es/service/utils.mjs.map +1 -0
- package/dist/es/task-runner.mjs +264 -0
- package/dist/es/task-runner.mjs.map +1 -0
- package/dist/es/tree.mjs +13 -2
- package/dist/es/tree.mjs.map +1 -0
- package/dist/es/types.mjs +18 -1
- package/dist/es/types.mjs.map +1 -1
- package/dist/es/utils.mjs +6 -7
- package/dist/es/utils.mjs.map +1 -1
- package/dist/es/yaml/builder.mjs.map +1 -1
- package/dist/es/yaml/player.mjs +121 -98
- package/dist/es/yaml/player.mjs.map +1 -1
- package/dist/es/yaml/utils.mjs +1 -1
- package/dist/es/yaml/utils.mjs.map +1 -1
- package/dist/lib/agent/agent.js +231 -142
- package/dist/lib/agent/agent.js.map +1 -1
- package/dist/lib/agent/common.js +1 -1
- package/dist/lib/agent/execution-session.js +75 -0
- package/dist/lib/agent/execution-session.js.map +1 -0
- package/dist/lib/agent/index.js +14 -14
- package/dist/lib/agent/index.js.map +1 -1
- package/dist/lib/agent/task-builder.js +356 -0
- package/dist/lib/agent/task-builder.js.map +1 -0
- package/dist/lib/agent/task-cache.js +8 -8
- package/dist/lib/agent/task-cache.js.map +1 -1
- package/dist/lib/agent/tasks.js +202 -506
- package/dist/lib/agent/tasks.js.map +1 -1
- package/dist/lib/agent/ui-utils.js +58 -36
- package/dist/lib/agent/ui-utils.js.map +1 -1
- package/dist/lib/agent/utils.js +26 -68
- package/dist/lib/agent/utils.js.map +1 -1
- package/dist/lib/ai-model/conversation-history.js +27 -15
- package/dist/lib/ai-model/conversation-history.js.map +1 -1
- package/dist/lib/ai-model/index.js +27 -27
- package/dist/lib/ai-model/index.js.map +1 -1
- package/dist/lib/ai-model/inspect.js +51 -57
- package/dist/lib/ai-model/inspect.js.map +1 -1
- package/dist/lib/ai-model/llm-planning.js +49 -67
- package/dist/lib/ai-model/llm-planning.js.map +1 -1
- package/dist/lib/ai-model/prompt/assertion.js +2 -2
- package/dist/lib/ai-model/prompt/assertion.js.map +1 -1
- package/dist/lib/ai-model/prompt/common.js +2 -2
- package/dist/lib/ai-model/prompt/common.js.map +1 -1
- package/dist/lib/ai-model/prompt/describe.js +2 -2
- package/dist/lib/ai-model/prompt/describe.js.map +1 -1
- package/dist/lib/ai-model/prompt/extraction.js +2 -2
- package/dist/lib/ai-model/prompt/extraction.js.map +1 -1
- package/dist/lib/ai-model/prompt/llm-locator.js +14 -241
- package/dist/lib/ai-model/prompt/llm-locator.js.map +1 -1
- package/dist/lib/ai-model/prompt/llm-planning.js +79 -328
- package/dist/lib/ai-model/prompt/llm-planning.js.map +1 -1
- package/dist/lib/ai-model/prompt/llm-section-locator.js +17 -16
- package/dist/lib/ai-model/prompt/llm-section-locator.js.map +1 -1
- package/dist/lib/ai-model/prompt/order-sensitive-judge.js +72 -0
- package/dist/lib/ai-model/prompt/order-sensitive-judge.js.map +1 -0
- package/dist/lib/ai-model/prompt/playwright-generator.js +11 -11
- package/dist/lib/ai-model/prompt/playwright-generator.js.map +1 -1
- package/dist/lib/ai-model/prompt/ui-tars-locator.js +2 -2
- package/dist/lib/ai-model/prompt/ui-tars-locator.js.map +1 -1
- package/dist/lib/ai-model/prompt/ui-tars-planning.js +2 -2
- package/dist/lib/ai-model/prompt/ui-tars-planning.js.map +1 -1
- package/dist/lib/ai-model/prompt/util.js +7 -95
- package/dist/lib/ai-model/prompt/util.js.map +1 -1
- package/dist/lib/ai-model/prompt/yaml-generator.js +18 -18
- package/dist/lib/ai-model/prompt/yaml-generator.js.map +1 -1
- package/dist/lib/ai-model/service-caller/index.js +288 -401
- package/dist/lib/ai-model/service-caller/index.js.map +1 -1
- package/dist/lib/ai-model/ui-tars-planning.js +71 -10
- package/dist/lib/ai-model/ui-tars-planning.js.map +1 -1
- package/dist/lib/{ai-model/common.js → common.js} +40 -55
- package/dist/lib/common.js.map +1 -0
- package/dist/lib/device/device-options.js +20 -0
- package/dist/lib/device/device-options.js.map +1 -0
- package/dist/lib/device/index.js +63 -40
- package/dist/lib/device/index.js.map +1 -1
- package/dist/lib/image/index.js +5 -5
- package/dist/lib/image/index.js.map +1 -1
- package/dist/lib/index.js +24 -20
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/report.js +2 -2
- package/dist/lib/report.js.map +1 -1
- package/dist/lib/{insight → service}/index.js +41 -54
- package/dist/lib/service/index.js.map +1 -0
- package/dist/lib/{insight → service}/utils.js +7 -7
- package/dist/lib/service/utils.js.map +1 -0
- package/dist/lib/task-runner.js +301 -0
- package/dist/lib/task-runner.js.map +1 -0
- package/dist/lib/tree.js +13 -4
- package/dist/lib/tree.js.map +1 -1
- package/dist/lib/types.js +31 -12
- package/dist/lib/types.js.map +1 -1
- package/dist/lib/utils.js +16 -17
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml/builder.js +2 -2
- package/dist/lib/yaml/builder.js.map +1 -1
- package/dist/lib/yaml/index.js +16 -22
- package/dist/lib/yaml/index.js.map +1 -1
- package/dist/lib/yaml/player.js +123 -100
- package/dist/lib/yaml/player.js.map +1 -1
- package/dist/lib/yaml/utils.js +6 -6
- package/dist/lib/yaml/utils.js.map +1 -1
- package/dist/lib/yaml.js +1 -1
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent/agent.d.ts +62 -17
- package/dist/types/agent/execution-session.d.ts +36 -0
- package/dist/types/agent/index.d.ts +3 -2
- package/dist/types/agent/task-builder.d.ts +35 -0
- package/dist/types/agent/tasks.d.ts +32 -23
- package/dist/types/agent/ui-utils.d.ts +9 -2
- package/dist/types/agent/utils.d.ts +9 -35
- package/dist/types/ai-model/conversation-history.d.ts +8 -4
- package/dist/types/ai-model/index.d.ts +5 -5
- package/dist/types/ai-model/inspect.d.ts +20 -12
- package/dist/types/ai-model/llm-planning.d.ts +3 -1
- package/dist/types/ai-model/prompt/llm-locator.d.ts +1 -6
- package/dist/types/ai-model/prompt/llm-planning.d.ts +2 -3
- package/dist/types/ai-model/prompt/llm-section-locator.d.ts +1 -3
- package/dist/types/ai-model/prompt/order-sensitive-judge.d.ts +2 -0
- package/dist/types/ai-model/prompt/util.d.ts +2 -34
- package/dist/types/ai-model/service-caller/index.d.ts +2 -3
- package/dist/types/ai-model/ui-tars-planning.d.ts +15 -2
- package/dist/types/{ai-model/common.d.ts → common.d.ts} +6 -6
- package/dist/types/device/device-options.d.ts +57 -0
- package/dist/types/device/index.d.ts +55 -39
- package/dist/types/index.d.ts +7 -6
- package/dist/types/service/index.d.ts +26 -0
- package/dist/types/service/utils.d.ts +2 -0
- package/dist/types/task-runner.d.ts +49 -0
- package/dist/types/tree.d.ts +4 -1
- package/dist/types/types.d.ts +103 -66
- package/dist/types/yaml/utils.d.ts +1 -1
- package/dist/types/yaml.d.ts +68 -43
- package/package.json +9 -12
- package/dist/es/ai-model/action-executor.mjs +0 -129
- package/dist/es/ai-model/action-executor.mjs.map +0 -1
- package/dist/es/ai-model/common.mjs.map +0 -1
- package/dist/es/insight/index.mjs.map +0 -1
- package/dist/es/insight/utils.mjs.map +0 -1
- package/dist/lib/ai-model/action-executor.js +0 -163
- package/dist/lib/ai-model/action-executor.js.map +0 -1
- package/dist/lib/ai-model/common.js.map +0 -1
- package/dist/lib/insight/index.js.map +0 -1
- package/dist/lib/insight/utils.js.map +0 -1
- package/dist/types/ai-model/action-executor.d.ts +0 -19
- package/dist/types/insight/index.d.ts +0 -31
- package/dist/types/insight/utils.d.ts +0 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ai-model/prompt/yaml-generator.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/prompt/yaml-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n StreamingAIResponse,\n StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n AIActionType,\n type ChatCompletionMessageParam,\n callAI,\n callAIWithStringResponse,\n} from '../index';\n\n// Common interfaces for test generation (shared between YAML and Playwright)\nexport interface EventCounts {\n navigation: number;\n click: number;\n input: number;\n scroll: number;\n total: number;\n}\n\nexport interface InputDescription {\n description: string;\n value: string;\n}\n\nexport interface ProcessedEvent {\n type: string;\n timestamp: number;\n url?: string;\n title?: string;\n elementDescription?: string;\n value?: string;\n pageInfo?: any;\n elementRect?: any;\n}\n\nexport interface EventSummary {\n testName: string;\n startUrl: string;\n eventCounts: EventCounts;\n urls: string[];\n clickDescriptions: string[];\n inputDescriptions: InputDescription[];\n events: ProcessedEvent[];\n}\n\n// Common ChromeRecordedEvent interface\nexport interface ChromeRecordedEvent {\n type: string;\n timestamp: number;\n url?: string;\n title?: string;\n elementDescription?: string;\n value?: string;\n pageInfo?: any;\n elementRect?: any;\n screenshotBefore?: string;\n screenshotAfter?: string;\n screenshotWithBox?: string;\n}\n\nexport interface YamlGenerationOptions {\n testName?: string;\n includeTimestamps?: boolean;\n maxScreenshots?: number;\n description?: string;\n}\n\nexport interface FilteredEvents {\n navigationEvents: ChromeRecordedEvent[];\n clickEvents: ChromeRecordedEvent[];\n inputEvents: ChromeRecordedEvent[];\n scrollEvents: ChromeRecordedEvent[];\n}\n\n// Common utility functions (shared between YAML and Playwright generators)\n\n/**\n * Get screenshots from events for LLM context\n */\nexport const getScreenshotsForLLM = (\n events: ChromeRecordedEvent[],\n maxScreenshots = 1,\n): string[] => {\n // Find events with screenshots, prioritizing navigation and click events\n const eventsWithScreenshots = events.filter(\n (event) =>\n event.screenshotBefore ||\n event.screenshotAfter ||\n event.screenshotWithBox,\n );\n\n // Sort them by priority (navigation first, then clicks, then others)\n const sortedEvents = [...eventsWithScreenshots].sort((a, b) => {\n if (a.type === 'navigation' && b.type !== 'navigation') return -1;\n if (a.type !== 'navigation' && b.type === 'navigation') return 1;\n if (a.type === 'click' && b.type !== 'click') return -1;\n if (a.type !== 'click' && b.type === 'click') return 1;\n return 0;\n });\n\n // Extract up to maxScreenshots screenshots\n const screenshots: string[] = [];\n for (const event of sortedEvents) {\n // Prefer the most informative screenshot\n const screenshot =\n event.screenshotWithBox ||\n event.screenshotAfter ||\n event.screenshotBefore;\n if (screenshot && !screenshots.includes(screenshot)) {\n screenshots.push(screenshot);\n if (screenshots.length >= maxScreenshots) break;\n }\n }\n\n return screenshots;\n};\n\n/**\n * Filter events by type for easier processing\n */\nexport const filterEventsByType = (\n events: ChromeRecordedEvent[],\n): FilteredEvents => {\n return {\n navigationEvents: events.filter((event) => event.type === 'navigation'),\n clickEvents: events.filter((event) => event.type === 'click'),\n inputEvents: events.filter((event) => event.type === 'input'),\n scrollEvents: events.filter((event) => event.type === 'scroll'),\n };\n};\n\n/**\n * Create event counts summary\n */\nexport const createEventCounts = (\n filteredEvents: FilteredEvents,\n totalEvents: number,\n): EventCounts => {\n return {\n navigation: filteredEvents.navigationEvents.length,\n click: filteredEvents.clickEvents.length,\n input: filteredEvents.inputEvents.length,\n scroll: filteredEvents.scrollEvents.length,\n total: totalEvents,\n };\n};\n\n/**\n * Extract input descriptions from input events\n */\nexport const extractInputDescriptions = (\n inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n return inputEvents\n .map((event) => ({\n description: event.elementDescription || '',\n value: event.value || '',\n }))\n .filter((item) => item.description && item.value);\n};\n\n/**\n * Process events for LLM consumption\n */\nexport const processEventsForLLM = (\n events: ChromeRecordedEvent[],\n): ProcessedEvent[] => {\n return events.map((event) => ({\n type: event.type,\n timestamp: event.timestamp,\n url: event.url,\n title: event.title,\n elementDescription: event.elementDescription,\n value: event.value,\n pageInfo: event.pageInfo,\n elementRect: event.elementRect,\n }));\n};\n\n/**\n * Prepare comprehensive event summary for LLM\n */\nexport const prepareEventSummary = (\n events: ChromeRecordedEvent[],\n options: { testName?: string; maxScreenshots?: number } = {},\n): EventSummary => {\n const filteredEvents = filterEventsByType(events);\n const eventCounts = createEventCounts(filteredEvents, events.length);\n\n // Extract useful information from events\n const startUrl =\n filteredEvents.navigationEvents.length > 0\n ? filteredEvents.navigationEvents[0].url || ''\n : '';\n\n const clickDescriptions = filteredEvents.clickEvents\n .map((event) => event.elementDescription)\n .filter((desc): desc is string => Boolean(desc))\n .slice(0, 10);\n\n const inputDescriptions = extractInputDescriptions(\n filteredEvents.inputEvents,\n ).slice(0, 10);\n\n const urls = filteredEvents.navigationEvents\n .map((e) => e.url)\n .filter((url): url is string => Boolean(url))\n .slice(0, 5);\n\n const processedEvents = processEventsForLLM(events);\n\n return {\n testName: options.testName || 'Automated test from recorded events',\n startUrl,\n eventCounts,\n urls,\n clickDescriptions,\n inputDescriptions,\n events: processedEvents,\n };\n};\n\n/**\n * Create message content for LLM with optional screenshots\n */\nexport const createMessageContent = (\n promptText: string,\n screenshots: string[] = [],\n includeScreenshots = true,\n) => {\n const messageContent: any[] = [\n {\n type: 'text',\n text: promptText,\n },\n ];\n\n // Add screenshots if available and requested\n if (includeScreenshots && screenshots.length > 0) {\n messageContent.unshift({\n type: 'text',\n text: 'Here are screenshots from the recording session to help you understand the context:',\n });\n\n screenshots.forEach((screenshot) => {\n messageContent.push({\n type: 'image_url',\n image_url: {\n url: screenshot,\n },\n });\n });\n }\n\n return messageContent;\n};\n\n/**\n * Validate events before processing\n */\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n if (!events.length) {\n throw new Error('No events provided for test generation');\n }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n events: ChromeRecordedEvent[],\n options: YamlGenerationOptions,\n modelConfig: IModelConfig,\n): Promise<string> => {\n try {\n // Validate input\n validateEvents(events);\n\n // Prepare event summary using shared utilities\n const summary = prepareEventSummary(events, {\n testName: options.testName,\n maxScreenshots: options.maxScreenshots || 3,\n });\n\n // Add YAML-specific options to summary\n const yamlSummary = {\n ...summary,\n includeTimestamps: options.includeTimestamps || false,\n };\n\n // Get screenshots for visual context\n const screenshots = getScreenshotsForLLM(\n events,\n options.maxScreenshots || 3,\n );\n\n // Use LLM to generate the YAML test configuration\n const prompt: ChatCompletionMessageParam[] = [\n {\n role: 'system',\n content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n },\n {\n role: 'user',\n content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n },\n ];\n\n // Add screenshots if available and requested\n if (screenshots.length > 0) {\n prompt.push({\n role: 'user',\n content:\n 'Here are screenshots from the recording session to help you understand the context:',\n });\n\n prompt.push({\n role: 'user',\n content: screenshots.map((screenshot) => ({\n type: 'image_url',\n image_url: {\n url: screenshot,\n },\n })),\n });\n }\n\n const response = await callAIWithStringResponse(\n prompt,\n AIActionType.TEXT,\n modelConfig,\n );\n\n if (response?.content && typeof response.content === 'string') {\n return response.content;\n }\n\n throw new Error('Failed to generate YAML test configuration');\n } catch (error) {\n throw new Error(`Failed to generate YAML test: ${error}`);\n }\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n events: ChromeRecordedEvent[],\n options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n try {\n // Validate input\n validateEvents(events);\n\n // Prepare event summary using shared utilities\n const summary = prepareEventSummary(events, {\n testName: options.testName,\n maxScreenshots: options.maxScreenshots || 3,\n });\n\n // Add YAML-specific options to summary\n const yamlSummary = {\n ...summary,\n includeTimestamps: options.includeTimestamps || false,\n };\n\n // Get screenshots for visual context\n const screenshots = getScreenshotsForLLM(\n events,\n options.maxScreenshots || 3,\n );\n\n // Use LLM to generate the YAML test configuration\n const prompt: ChatCompletionMessageParam[] = [\n {\n role: 'system',\n content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n },\n {\n role: 'user',\n content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n },\n ];\n\n // Add screenshots if available and requested\n if (screenshots.length > 0) {\n prompt.push({\n role: 'user',\n content:\n 'Here are screenshots from the recording session to help you understand the context:',\n });\n\n prompt.push({\n role: 'user',\n content: screenshots.map((screenshot) => ({\n type: 'image_url',\n image_url: {\n url: screenshot,\n },\n })),\n });\n }\n\n if (options.stream && options.onChunk) {\n // Use streaming\n return await callAI(prompt, AIActionType.TEXT, modelConfig, {\n stream: true,\n onChunk: options.onChunk,\n });\n } else {\n // Fallback to non-streaming\n const response = await callAIWithStringResponse(\n prompt,\n AIActionType.TEXT,\n modelConfig,\n );\n\n if (response?.content && typeof response.content === 'string') {\n return {\n content: response.content,\n usage: response.usage,\n isStreamed: false,\n };\n }\n\n throw new Error('Failed to generate YAML test configuration');\n }\n } catch (error) {\n throw new Error(`Failed to generate YAML test: ${error}`);\n }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getScreenshotsForLLM","events","maxScreenshots","eventsWithScreenshots","event","sortedEvents","a","b","screenshots","screenshot","filterEventsByType","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","item","processEventsForLLM","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","Boolean","inputDescriptions","urls","e","url","processedEvents","createMessageContent","promptText","includeScreenshots","messageContent","validateEvents","Error","generateYamlTest","modelConfig","summary","yamlSummary","prompt","YAML_EXAMPLE_CODE","JSON","response","callAIWithStringResponse","AIActionType","error","generateYamlTestStream","callAI"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;AC4EO,MAAMI,uBAAuB,CAClCC,QACAC,iBAAiB,CAAC;IAGlB,MAAMC,wBAAwBF,OAAO,MAAM,CACzC,CAACG,QACCA,MAAM,gBAAgB,IACtBA,MAAM,eAAe,IACrBA,MAAM,iBAAiB;IAI3B,MAAMC,eAAe;WAAIF;KAAsB,CAAC,IAAI,CAAC,CAACG,GAAGC;QACvD,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,OAAO;IACT;IAGA,MAAMC,cAAwB,EAAE;IAChC,KAAK,MAAMJ,SAASC,aAAc;QAEhC,MAAMI,aACJL,MAAM,iBAAiB,IACvBA,MAAM,eAAe,IACrBA,MAAM,gBAAgB;QACxB,IAAIK,cAAc,CAACD,YAAY,QAAQ,CAACC,aAAa;YACnDD,YAAY,IAAI,CAACC;YACjB,IAAID,YAAY,MAAM,IAAIN,gBAAgB;QAC5C;IACF;IAEA,OAAOM;AACT;AAKO,MAAME,qBAAqB,CAChCT,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAMK,MAAMO,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAMK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACX,QAAW;YACf,aAAaA,MAAM,kBAAkB,IAAI;YACzC,OAAOA,MAAM,KAAK,IAAI;QACxB,IACC,MAAM,CAAC,CAACY,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAM7C,MAAMC,sBAAsB,CACjChB,SAEOA,OAAO,GAAG,CAAC,CAACG,QAAW;YAC5B,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClB,oBAAoBA,MAAM,kBAAkB;YAC5C,OAAOA,MAAM,KAAK;YAClB,UAAUA,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;QAChC;AAMK,MAAMc,sBAAsB,CACjCjB,QACAkB,UAA0D,CAAC,CAAC;IAE5D,MAAMP,iBAAiBF,mBAAmBT;IAC1C,MAAMmB,cAAcT,kBAAkBC,gBAAgBX,OAAO,MAAM;IAGnE,MAAMoB,WACJT,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAMU,oBAAoBV,eAAe,WAAW,CACjD,GAAG,CAAC,CAACR,QAAUA,MAAM,kBAAkB,EACvC,MAAM,CAAC,CAACmB,OAAyBC,QAAQD,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAME,oBAAoBX,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAMc,OAAOd,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAACe,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBJ,QAAQI,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkBZ,oBAAoBhB;IAE5C,OAAO;QACL,UAAUkB,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAM;QACAJ;QACAG;QACA,QAAQI;IACV;AACF;AAKO,MAAMC,uBAAuB,CAClCC,YACAvB,cAAwB,EAAE,EAC1BwB,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMF;QACR;KACD;IAGD,IAAIC,sBAAsBxB,YAAY,MAAM,GAAG,GAAG;QAChDyB,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAzB,YAAY,OAAO,CAAC,CAACC;YACnBwB,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKxB;gBACP;YACF;QACF;IACF;IAEA,OAAOwB;AACT;AAKO,MAAMC,iBAAiB,CAACjC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIkC,MAAM;AAEpB;AAOO,MAAMC,mBAAmB,OAC9BnC,QACAkB,SACAkB;IAEA,IAAI;QAEFH,eAAejC;QAGf,MAAMqC,UAAUpB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMoB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBnB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMqB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;8JASuH,CAAC;YACzJ;SACD;QAGD,IAAI/B,YAAY,MAAM,GAAG,GAAG;YAC1BgC,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAShC,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,MAAMkC,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACrBJ,QACAK,kCAAAA,YAAAA,CAAAA,IAAiB,EACjBR;QAGF,IAAIM,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;QAGzB,MAAM,IAAIR,MAAM;IAClB,EAAE,OAAOW,OAAO;QACd,MAAM,IAAIX,MAAM,CAAC,8BAA8B,EAAEW,OAAO;IAC1D;AACF;AAKO,MAAMC,yBAAyB,OACpC9C,QACAkB,SACAkB;IAEA,IAAI;QAEFH,eAAejC;QAGf,MAAMqC,UAAUpB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMoB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBnB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMqB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;8JASuH,CAAC;YACzJ;SACD;QAGD,IAAI/B,YAAY,MAAM,GAAG,GAAG;YAC1BgC,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAShC,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,IAAIU,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAM6B,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOR,QAAQK,kCAAAA,YAAAA,CAAAA,IAAiB,EAAER,aAAa;YAC1D,QAAQ;YACR,SAASlB,QAAQ,OAAO;QAC1B;QACK;YAEL,MAAMwB,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACrBJ,QACAK,kCAAAA,YAAAA,CAAAA,IAAiB,EACjBR;YAGF,IAAIM,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;gBACL,SAASA,SAAS,OAAO;gBACzB,OAAOA,SAAS,KAAK;gBACrB,YAAY;YACd;YAGF,MAAM,IAAIR,MAAM;QAClB;IACF,EAAE,OAAOW,OAAO;QACd,MAAM,IAAIX,MAAM,CAAC,8BAA8B,EAAEW,OAAO;IAC1D;AACF"}
|
|
1
|
+
{"version":3,"file":"ai-model/prompt/yaml-generator.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/yaml-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n StreamingAIResponse,\n StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n AIActionType,\n type ChatCompletionMessageParam,\n callAI,\n callAIWithStringResponse,\n} from '../index';\n\n// Common interfaces for test generation (shared between YAML and Playwright)\nexport interface EventCounts {\n navigation: number;\n click: number;\n input: number;\n scroll: number;\n total: number;\n}\n\nexport interface InputDescription {\n description: string;\n value: string;\n}\n\nexport interface ProcessedEvent {\n type: string;\n timestamp: number;\n url?: string;\n title?: string;\n elementDescription?: string;\n value?: string;\n pageInfo?: any;\n elementRect?: any;\n}\n\nexport interface EventSummary {\n testName: string;\n startUrl: string;\n eventCounts: EventCounts;\n urls: string[];\n clickDescriptions: string[];\n inputDescriptions: InputDescription[];\n events: ProcessedEvent[];\n}\n\n// Common ChromeRecordedEvent interface\nexport interface ChromeRecordedEvent {\n type: string;\n timestamp: number;\n url?: string;\n title?: string;\n elementDescription?: string;\n value?: string;\n pageInfo?: any;\n elementRect?: any;\n screenshotBefore?: string;\n screenshotAfter?: string;\n screenshotWithBox?: string;\n}\n\nexport interface YamlGenerationOptions {\n testName?: string;\n includeTimestamps?: boolean;\n maxScreenshots?: number;\n description?: string;\n}\n\nexport interface FilteredEvents {\n navigationEvents: ChromeRecordedEvent[];\n clickEvents: ChromeRecordedEvent[];\n inputEvents: ChromeRecordedEvent[];\n scrollEvents: ChromeRecordedEvent[];\n}\n\n// Common utility functions (shared between YAML and Playwright generators)\n\n/**\n * Get screenshots from events for LLM context\n */\nexport const getScreenshotsForLLM = (\n events: ChromeRecordedEvent[],\n maxScreenshots = 1,\n): string[] => {\n // Find events with screenshots, prioritizing navigation and click events\n const eventsWithScreenshots = events.filter(\n (event) =>\n event.screenshotBefore ||\n event.screenshotAfter ||\n event.screenshotWithBox,\n );\n\n // Sort them by priority (navigation first, then clicks, then others)\n const sortedEvents = [...eventsWithScreenshots].sort((a, b) => {\n if (a.type === 'navigation' && b.type !== 'navigation') return -1;\n if (a.type !== 'navigation' && b.type === 'navigation') return 1;\n if (a.type === 'click' && b.type !== 'click') return -1;\n if (a.type !== 'click' && b.type === 'click') return 1;\n return 0;\n });\n\n // Extract up to maxScreenshots screenshots\n const screenshots: string[] = [];\n for (const event of sortedEvents) {\n // Prefer the most informative screenshot\n const screenshot =\n event.screenshotWithBox ||\n event.screenshotAfter ||\n event.screenshotBefore;\n if (screenshot && !screenshots.includes(screenshot)) {\n screenshots.push(screenshot);\n if (screenshots.length >= maxScreenshots) break;\n }\n }\n\n return screenshots;\n};\n\n/**\n * Filter events by type for easier processing\n */\nexport const filterEventsByType = (\n events: ChromeRecordedEvent[],\n): FilteredEvents => {\n return {\n navigationEvents: events.filter((event) => event.type === 'navigation'),\n clickEvents: events.filter((event) => event.type === 'click'),\n inputEvents: events.filter((event) => event.type === 'input'),\n scrollEvents: events.filter((event) => event.type === 'scroll'),\n };\n};\n\n/**\n * Create event counts summary\n */\nexport const createEventCounts = (\n filteredEvents: FilteredEvents,\n totalEvents: number,\n): EventCounts => {\n return {\n navigation: filteredEvents.navigationEvents.length,\n click: filteredEvents.clickEvents.length,\n input: filteredEvents.inputEvents.length,\n scroll: filteredEvents.scrollEvents.length,\n total: totalEvents,\n };\n};\n\n/**\n * Extract input descriptions from input events\n */\nexport const extractInputDescriptions = (\n inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n return inputEvents\n .map((event) => ({\n description: event.elementDescription || '',\n value: event.value || '',\n }))\n .filter((item) => item.description && item.value);\n};\n\n/**\n * Process events for LLM consumption\n */\nexport const processEventsForLLM = (\n events: ChromeRecordedEvent[],\n): ProcessedEvent[] => {\n return events.map((event) => ({\n type: event.type,\n timestamp: event.timestamp,\n url: event.url,\n title: event.title,\n elementDescription: event.elementDescription,\n value: event.value,\n pageInfo: event.pageInfo,\n elementRect: event.elementRect,\n }));\n};\n\n/**\n * Prepare comprehensive event summary for LLM\n */\nexport const prepareEventSummary = (\n events: ChromeRecordedEvent[],\n options: { testName?: string; maxScreenshots?: number } = {},\n): EventSummary => {\n const filteredEvents = filterEventsByType(events);\n const eventCounts = createEventCounts(filteredEvents, events.length);\n\n // Extract useful information from events\n const startUrl =\n filteredEvents.navigationEvents.length > 0\n ? filteredEvents.navigationEvents[0].url || ''\n : '';\n\n const clickDescriptions = filteredEvents.clickEvents\n .map((event) => event.elementDescription)\n .filter((desc): desc is string => Boolean(desc))\n .slice(0, 10);\n\n const inputDescriptions = extractInputDescriptions(\n filteredEvents.inputEvents,\n ).slice(0, 10);\n\n const urls = filteredEvents.navigationEvents\n .map((e) => e.url)\n .filter((url): url is string => Boolean(url))\n .slice(0, 5);\n\n const processedEvents = processEventsForLLM(events);\n\n return {\n testName: options.testName || 'Automated test from recorded events',\n startUrl,\n eventCounts,\n urls,\n clickDescriptions,\n inputDescriptions,\n events: processedEvents,\n };\n};\n\n/**\n * Create message content for LLM with optional screenshots\n */\nexport const createMessageContent = (\n promptText: string,\n screenshots: string[] = [],\n includeScreenshots = true,\n) => {\n const messageContent: any[] = [\n {\n type: 'text',\n text: promptText,\n },\n ];\n\n // Add screenshots if available and requested\n if (includeScreenshots && screenshots.length > 0) {\n messageContent.unshift({\n type: 'text',\n text: 'Here are screenshots from the recording session to help you understand the context:',\n });\n\n screenshots.forEach((screenshot) => {\n messageContent.push({\n type: 'image_url',\n image_url: {\n url: screenshot,\n },\n });\n });\n }\n\n return messageContent;\n};\n\n/**\n * Validate events before processing\n */\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n if (!events.length) {\n throw new Error('No events provided for test generation');\n }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n events: ChromeRecordedEvent[],\n options: YamlGenerationOptions,\n modelConfig: IModelConfig,\n): Promise<string> => {\n try {\n // Validate input\n validateEvents(events);\n\n // Prepare event summary using shared utilities\n const summary = prepareEventSummary(events, {\n testName: options.testName,\n maxScreenshots: options.maxScreenshots || 3,\n });\n\n // Add YAML-specific options to summary\n const yamlSummary = {\n ...summary,\n includeTimestamps: options.includeTimestamps || false,\n };\n\n // Get screenshots for visual context\n const screenshots = getScreenshotsForLLM(\n events,\n options.maxScreenshots || 3,\n );\n\n // Use LLM to generate the YAML test configuration\n const prompt: ChatCompletionMessageParam[] = [\n {\n role: 'system',\n content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n },\n {\n role: 'user',\n content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n },\n ];\n\n // Add screenshots if available and requested\n if (screenshots.length > 0) {\n prompt.push({\n role: 'user',\n content:\n 'Here are screenshots from the recording session to help you understand the context:',\n });\n\n prompt.push({\n role: 'user',\n content: screenshots.map((screenshot) => ({\n type: 'image_url',\n image_url: {\n url: screenshot,\n },\n })),\n });\n }\n\n const response = await callAIWithStringResponse(\n prompt,\n AIActionType.TEXT,\n modelConfig,\n );\n\n if (response?.content && typeof response.content === 'string') {\n return response.content;\n }\n\n throw new Error('Failed to generate YAML test configuration');\n } catch (error) {\n throw new Error(`Failed to generate YAML test: ${error}`);\n }\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n events: ChromeRecordedEvent[],\n options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n try {\n // Validate input\n validateEvents(events);\n\n // Prepare event summary using shared utilities\n const summary = prepareEventSummary(events, {\n testName: options.testName,\n maxScreenshots: options.maxScreenshots || 3,\n });\n\n // Add YAML-specific options to summary\n const yamlSummary = {\n ...summary,\n includeTimestamps: options.includeTimestamps || false,\n };\n\n // Get screenshots for visual context\n const screenshots = getScreenshotsForLLM(\n events,\n options.maxScreenshots || 3,\n );\n\n // Use LLM to generate the YAML test configuration\n const prompt: ChatCompletionMessageParam[] = [\n {\n role: 'system',\n content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n },\n {\n role: 'user',\n content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n },\n ];\n\n // Add screenshots if available and requested\n if (screenshots.length > 0) {\n prompt.push({\n role: 'user',\n content:\n 'Here are screenshots from the recording session to help you understand the context:',\n });\n\n prompt.push({\n role: 'user',\n content: screenshots.map((screenshot) => ({\n type: 'image_url',\n image_url: {\n url: screenshot,\n },\n })),\n });\n }\n\n if (options.stream && options.onChunk) {\n // Use streaming\n return await callAI(prompt, AIActionType.TEXT, modelConfig, {\n stream: true,\n onChunk: options.onChunk,\n });\n } else {\n // Fallback to non-streaming\n const response = await callAIWithStringResponse(\n prompt,\n AIActionType.TEXT,\n modelConfig,\n );\n\n if (response?.content && typeof response.content === 'string') {\n return {\n content: response.content,\n usage: response.usage,\n isStreamed: false,\n };\n }\n\n throw new Error('Failed to generate YAML test configuration');\n }\n } catch (error) {\n throw new Error(`Failed to generate YAML test: ${error}`);\n }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getScreenshotsForLLM","events","maxScreenshots","eventsWithScreenshots","event","sortedEvents","a","b","screenshots","screenshot","filterEventsByType","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","item","processEventsForLLM","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","Boolean","inputDescriptions","urls","e","url","processedEvents","createMessageContent","promptText","includeScreenshots","messageContent","validateEvents","Error","generateYamlTest","modelConfig","summary","yamlSummary","prompt","YAML_EXAMPLE_CODE","JSON","response","callAIWithStringResponse","AIActionType","error","generateYamlTestStream","callAI"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;AC4EO,MAAMI,uBAAuB,CAClCC,QACAC,iBAAiB,CAAC;IAGlB,MAAMC,wBAAwBF,OAAO,MAAM,CACzC,CAACG,QACCA,MAAM,gBAAgB,IACtBA,MAAM,eAAe,IACrBA,MAAM,iBAAiB;IAI3B,MAAMC,eAAe;WAAIF;KAAsB,CAAC,IAAI,CAAC,CAACG,GAAGC;QACvD,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,OAAO;IACT;IAGA,MAAMC,cAAwB,EAAE;IAChC,KAAK,MAAMJ,SAASC,aAAc;QAEhC,MAAMI,aACJL,MAAM,iBAAiB,IACvBA,MAAM,eAAe,IACrBA,MAAM,gBAAgB;QACxB,IAAIK,cAAc,CAACD,YAAY,QAAQ,CAACC,aAAa;YACnDD,YAAY,IAAI,CAACC;YACjB,IAAID,YAAY,MAAM,IAAIN,gBAAgB;QAC5C;IACF;IAEA,OAAOM;AACT;AAKO,MAAME,qBAAqB,CAChCT,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAMK,MAAMO,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAMK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACX,QAAW;YACf,aAAaA,MAAM,kBAAkB,IAAI;YACzC,OAAOA,MAAM,KAAK,IAAI;QACxB,IACC,MAAM,CAAC,CAACY,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAM7C,MAAMC,sBAAsB,CACjChB,SAEOA,OAAO,GAAG,CAAC,CAACG,QAAW;YAC5B,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClB,oBAAoBA,MAAM,kBAAkB;YAC5C,OAAOA,MAAM,KAAK;YAClB,UAAUA,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;QAChC;AAMK,MAAMc,sBAAsB,CACjCjB,QACAkB,UAA0D,CAAC,CAAC;IAE5D,MAAMP,iBAAiBF,mBAAmBT;IAC1C,MAAMmB,cAAcT,kBAAkBC,gBAAgBX,OAAO,MAAM;IAGnE,MAAMoB,WACJT,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAMU,oBAAoBV,eAAe,WAAW,CACjD,GAAG,CAAC,CAACR,QAAUA,MAAM,kBAAkB,EACvC,MAAM,CAAC,CAACmB,OAAyBC,QAAQD,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAME,oBAAoBX,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAMc,OAAOd,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAACe,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBJ,QAAQI,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkBZ,oBAAoBhB;IAE5C,OAAO;QACL,UAAUkB,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAM;QACAJ;QACAG;QACA,QAAQI;IACV;AACF;AAKO,MAAMC,uBAAuB,CAClCC,YACAvB,cAAwB,EAAE,EAC1BwB,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMF;QACR;KACD;IAGD,IAAIC,sBAAsBxB,YAAY,MAAM,GAAG,GAAG;QAChDyB,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAzB,YAAY,OAAO,CAAC,CAACC;YACnBwB,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKxB;gBACP;YACF;QACF;IACF;IAEA,OAAOwB;AACT;AAKO,MAAMC,iBAAiB,CAACjC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIkC,MAAM;AAEpB;AAOO,MAAMC,mBAAmB,OAC9BnC,QACAkB,SACAkB;IAEA,IAAI;QAEFH,eAAejC;QAGf,MAAMqC,UAAUpB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMoB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBnB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMqB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;8JASuH,CAAC;YACzJ;SACD;QAGD,IAAI/B,YAAY,MAAM,GAAG,GAAG;YAC1BgC,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAShC,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,MAAMkC,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACrBJ,QACAK,kCAAAA,YAAAA,CAAAA,IAAiB,EACjBR;QAGF,IAAIM,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;QAGzB,MAAM,IAAIR,MAAM;IAClB,EAAE,OAAOW,OAAO;QACd,MAAM,IAAIX,MAAM,CAAC,8BAA8B,EAAEW,OAAO;IAC1D;AACF;AAKO,MAAMC,yBAAyB,OACpC9C,QACAkB,SACAkB;IAEA,IAAI;QAEFH,eAAejC;QAGf,MAAMqC,UAAUpB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMoB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBnB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMqB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;8JASuH,CAAC;YACzJ;SACD;QAGD,IAAI/B,YAAY,MAAM,GAAG,GAAG;YAC1BgC,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAShC,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,IAAIU,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAM6B,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOR,QAAQK,kCAAAA,YAAAA,CAAAA,IAAiB,EAAER,aAAa;YAC1D,QAAQ;YACR,SAASlB,QAAQ,OAAO;QAC1B;QACK;YAEL,MAAMwB,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACrBJ,QACAK,kCAAAA,YAAAA,CAAAA,IAAiB,EACjBR;YAGF,IAAIM,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;gBACL,SAASA,SAAS,OAAO;gBACzB,OAAOA,SAAS,KAAK;gBACrB,YAAY;YACd;YAGF,MAAM,IAAIR,MAAM;QAClB;IACF,EAAE,OAAOW,OAAO;QACd,MAAM,IAAIX,MAAM,CAAC,8BAA8B,EAAEW,OAAO;IAC1D;AACF"}
|