@midscene/core 0.28.2 → 0.28.3-beta-20250910131004.0

package/dist/lib/ai-model/prompt/playwright-generator.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/playwright-generator.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/prompt/playwright-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { AIActionType, callAi } from '../index';\n\n// Import shared utilities and types from yaml-generator\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './yaml-generator';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions = {},\n): Promise<string> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nRespond ONLY with the complete Playwright test code, no explanations.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAi(prompt, AIActionType.EXTRACT_DATA, {\n    intent: 'default',\n  });\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions = {},\n): Promise<StreamingAIResponse> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nRespond ONLY with the complete Playwright test code, no explanations.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAi(\n      prompt,\n      AIActionType.EXTRACT_DATA,\n      {\n        intent: 'default',\n      },\n      {\n        stream: true,\n        onChunk: options.onChunk,\n      },\n    );\n  } else {\n    // Fallback to non-streaming\n    const response = await callAi(prompt, AIActionType.EXTRACT_DATA, {\n      intent: 'default',\n    });\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","generatePlaywrightTest","events","options","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAi","AIActionType","Error","generatePlaywrightTestStream"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;ACuDO,MAAMI,yBAAyB,OACpCC,QACAC,UAAuC,CAAC,CAAC;IAGzCC,IAAAA,2CAAAA,cAAAA,AAAAA,EAAeF;IAGf,MAAMG,UAAUC,AAAAA,IAAAA,2CAAAA,mBAAAA,AAAAA,EAAoBJ,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMI,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBF,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMK,cAAcC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EAAqBP,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMO,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;qEAWwB,CAAC;IAGpE,MAAMK,iBAAiBC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAL,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMW,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOF,QAAQG,kCAAAA,YAAAA,CAAAA,YAAyB,EAAE;QAC/D,QAAQ;IACV;IAEA,IAAIF,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIG,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CnB,QACAC,UAAwE,CAAC,CAAC;IAG1EC,IAAAA,2CAAAA,cAAAA,AAAAA,EAAeF;IAGf,MAAMG,UAAUC,AAAAA,IAAAA,2CAAAA,mBAAAA,AAAAA,EAAoBJ,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMI,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBF,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMK,cAAcC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EAAqBP,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMO,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;qEAYwB,CAAC;IAGpE,MAAMK,iBAAiBC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAL,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMW,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIT,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMe,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EACXF,QACAG,kCAAAA,YAAAA,CAAAA,YAAyB,EACzB;QACE,QAAQ;IACV,GACA;QACE,QAAQ;QACR,SAAShB,QAAQ,OAAO;IAC1B;IAEG;QAEL,MAAMc,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOF,QAAQG,kCAAAA,YAAAA,CAAAA,YAAyB,EAAE;YAC/D,QAAQ;QACV;QAEA,IAAIF,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIG,MAAM;IAClB;AACF"}
+{"version":3,"file":"ai-model/prompt/playwright-generator.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/prompt/playwright-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { AIActionType, callAI, callAIWithStringResponse } from '../index';\n// Import shared utilities and types from yaml-generator\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './yaml-generator';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nRespond ONLY with the complete Playwright test code, no explanations.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(\n    prompt,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nRespond ONLY with the complete Playwright test code, no explanations.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAI(prompt, AIActionType.EXTRACT_DATA, modelConfig, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(\n      prompt,\n      AIActionType.EXTRACT_DATA,\n      modelConfig,\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","generatePlaywrightTest","events","options","modelConfig","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","AIActionType","Error","generatePlaywrightTestStream","callAI"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;ACuDO,MAAMI,yBAAyB,OACpCC,QACAC,SACAC;IAGAC,IAAAA,2CAAAA,cAAAA,AAAAA,EAAeH;IAGf,MAAMI,UAAUC,AAAAA,IAAAA,2CAAAA,mBAAAA,AAAAA,EAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;qEAWwB,CAAC;IAGpE,MAAMK,iBAAiBC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACrBF,QACAG,kCAAAA,YAAAA,CAAAA,YAAyB,EACzBhB;IAGF,IAAIc,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIG,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CpB,QACAC,SACAC;IAGAC,IAAAA,2CAAAA,cAAAA,AAAAA,EAAeH;IAGf,MAAMI,UAAUC,AAAAA,IAAAA,2CAAAA,mBAAAA,AAAAA,EAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;qEAYwB,CAAC;IAGpE,MAAMK,iBAAiBC,AAAAA,IAAAA,2CAAAA,oBAAAA,AAAAA,EACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,0BAAAA,uBAAuBA,EAAE;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMoB,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAON,QAAQG,kCAAAA,YAAAA,CAAAA,YAAyB,EAAEhB,aAAa;QAClE,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACrBF,QACAG,kCAAAA,YAAAA,CAAAA,YAAyB,EACzBhB;QAGF,IAAIc,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIG,MAAM;IAClB;AACF"}

package/dist/lib/ai-model/prompt/util.js

@@ -34,7 +34,6 @@ __webpack_require__.d(__webpack_exports__, {
 });
 const index_js_namespaceObject = require("../../image/index.js");
 const constants_namespaceObject = require("@midscene/shared/constants");
-const env_namespaceObject = require("@midscene/shared/env");
 const extractor_namespaceObject = require("@midscene/shared/extractor");
 const utils_namespaceObject = require("@midscene/shared/utils");
 function describeSize(size) {
@@ -99,7 +98,7 @@ Description of all the elements in screenshot:
 </div>
 ====================
 `;
-async function describeUserPage(context, modelPreferences, opt) {
+async function describeUserPage(context, opt) {
     const { screenshotBase64 } = context;
     let width;
     let height;
@@ -118,7 +117,8 @@ async function describeUserPage(context, modelPreferences, opt) {
     });
     let pageDescription = '';
     const visibleOnly = (null == opt ? void 0 : opt.domIncluded) === 'visible-only' ? true : (null == opt ? void 0 : opt.visibleOnly) ?? false;
-    const shouldIncludeDOM = (null == opt ? void 0 : opt.domIncluded) || !(0, env_namespaceObject.vlLocateMode)(modelPreferences);
+    const resolvedVlMode = null == opt ? void 0 : opt.vlMode;
+    const shouldIncludeDOM = (null == opt ? void 0 : opt.domIncluded) || !resolvedVlMode;
     if (shouldIncludeDOM) {
         const contentTree = await (0, extractor_namespaceObject.descriptionOfTree)(treeRoot, null == opt ? void 0 : opt.truncateTextLength, null == opt ? void 0 : opt.filterNonTextContent, visibleOnly);
         const sizeDescription = describeSize({

package/dist/lib/ai-model/prompt/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/util.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/prompt/util.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { imageInfoOfBase64 } from '@/image/index';\nimport type { BaseElement, ElementTreeNode, Size, UIContext } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport { type IModelPreferences, vlLocateMode } from '@midscene/shared/env';\nimport {\n  descriptionOfTree,\n  generateElementByPosition,\n  treeToList,\n} from '@midscene/shared/extractor';\nimport { assert } from '@midscene/shared/utils';\n\nexport function describeSize(size: Size) {\n  return `${size.width} x ${size.height}`;\n}\n\nexport function describeElement(\n  elements: (Pick<BaseElement, 'rect' | 'content'> & { id: string })[],\n) {\n  const sliceLength = 80;\n  return elements\n    .map((item) =>\n      [\n        item.id,\n        item.rect.left,\n        item.rect.top,\n        item.rect.left + item.rect.width,\n        item.rect.top + item.rect.height,\n        item.content.length > sliceLength\n          ? `${item.content.slice(0, sliceLength)}...`\n          : item.content,\n      ].join(', '),\n    )\n    .join('\\n');\n}\nexport const distanceThreshold = 16;\n\nexport function elementByPositionWithElementInfo(\n  treeRoot: ElementTreeNode<BaseElement>,\n  position: {\n    x: number;\n    y: number;\n  },\n  options?: {\n    requireStrictDistance?: boolean;\n    filterPositionElements?: boolean;\n  },\n) {\n  const requireStrictDistance = options?.requireStrictDistance ?? true;\n  const filterPositionElements = options?.filterPositionElements ?? false;\n\n  assert(typeof position !== 'undefined', 'position is required for query');\n\n  const matchingElements: BaseElement[] = [];\n\n  function dfs(node: ElementTreeNode<BaseElement>) {\n    if (node?.node) {\n      const item = node.node;\n      if (\n        item.rect.left <= position.x &&\n        position.x <= item.rect.left + item.rect.width &&\n        item.rect.top <= position.y &&\n        position.y <= item.rect.top + item.rect.height\n      ) {\n        if (\n          !(\n            filterPositionElements &&\n            item.attributes?.nodeType === NodeType.POSITION\n          ) &&\n          item.isVisible\n        ) {\n          matchingElements.push(item);\n        }\n      }\n    }\n\n    for (const child of node.children) {\n      dfs(child);\n    }\n  }\n\n  dfs(treeRoot);\n\n  if (matchingElements.length === 0) {\n    return undefined;\n  }\n\n  // Find the smallest element by area\n  const element = matchingElements.reduce((smallest, current) => {\n    const smallestArea = smallest.rect.width * smallest.rect.height;\n    const currentArea = current.rect.width * current.rect.height;\n    return currentArea < smallestArea ? current : smallest;\n  });\n\n  const distanceToCenter = distance(\n    { x: element.center[0], y: element.center[1] },\n    position,\n  );\n\n  if (requireStrictDistance) {\n    return distanceToCenter <= distanceThreshold ? element : undefined;\n  }\n\n  return element;\n}\n\nexport function distance(\n  point1: { x: number; y: number },\n  point2: { x: number; y: number },\n) {\n  return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);\n}\n\nexport const samplePageDescription = `\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n`;\n\nexport async function describeUserPage<\n  ElementType extends BaseElement = BaseElement,\n>(\n  context: Omit<UIContext<ElementType>, 'describer'>,\n  modelPreferences: IModelPreferences,\n  opt?: {\n    truncateTextLength?: number;\n    filterNonTextContent?: boolean;\n    domIncluded?: boolean | 'visible-only';\n    visibleOnly?: boolean;\n  },\n) {\n  const { screenshotBase64 } = context;\n  let width: number;\n  let height: number;\n\n  if (context.size) {\n    ({ width, height } = context.size);\n  } else {\n    const imgSize = await imageInfoOfBase64(screenshotBase64);\n    ({ width, height } = imgSize);\n  }\n\n  const treeRoot = context.tree;\n  // dfs tree, save the id and element info\n  const idElementMap: Record<string, ElementType> = {};\n  const flatElements: ElementType[] = treeToList(treeRoot);\n\n  if (opt?.domIncluded === true && flatElements.length >= 5000) {\n    console.warn(\n      'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: \"visible-only\" to reduce the number of elements',\n    );\n  }\n\n  flatElements.forEach((element) => {\n    idElementMap[element.id] = element;\n    if (typeof element.indexId !== 'undefined') {\n      idElementMap[`${element.indexId}`] = element;\n    }\n  });\n\n  let pageDescription = '';\n\n  const visibleOnly =\n    opt?.domIncluded === 'visible-only' ? true : (opt?.visibleOnly ?? false);\n  const shouldIncludeDOM = opt?.domIncluded || !vlLocateMode(modelPreferences);\n\n  if (shouldIncludeDOM) {\n    // non-vl mode must provide the page description\n    const contentTree = await descriptionOfTree(\n      treeRoot,\n      opt?.truncateTextLength,\n      opt?.filterNonTextContent,\n      visibleOnly,\n    );\n\n    // if match by position, don't need to provide the page description\n    const sizeDescription = describeSize({ width, height });\n    pageDescription = `The size of the page: ${sizeDescription} \\n The page elements tree:\\n${contentTree}`;\n  }\n\n  return {\n    description: pageDescription,\n    elementById(idOrIndexId: string) {\n      assert(typeof idOrIndexId !== 'undefined', 'id is required for query');\n      const item = idElementMap[`${idOrIndexId}`];\n      return item;\n    },\n    elementByPosition(\n      position: { x: number; y: number },\n      size: { width: number; height: number },\n    ) {\n      return elementByPositionWithElementInfo(treeRoot, position);\n    },\n    insertElementByPosition(position: { x: number; y: number }) {\n      const element = generateElementByPosition(position) as ElementType;\n\n      treeRoot.children.push({\n        node: element,\n        children: [],\n      });\n      flatElements.push(element);\n      idElementMap[element.id] = element;\n      return element;\n    },\n    size: { width, height },\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","describeSize","size","describeElement","elements","sliceLength","item","distanceThreshold","elementByPositionWithElementInfo","treeRoot","position","options","requireStrictDistance","filterPositionElements","assert","matchingElements","dfs","node","_item_attributes","NodeType","child","element","smallest","current","smallestArea","currentArea","distanceToCenter","distance","undefined","point1","point2","Math","samplePageDescription","describeUserPage","context","modelPreferences","opt","screenshotBase64","width","height","imgSize","imageInfoOfBase64","idElementMap","flatElements","treeToList","console","pageDescription","visibleOnly","shouldIncludeDOM","vlLocateMode","contentTree","descriptionOfTree","sizeDescription","idOrIndexId","generateElementByPosition"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;ACKO,SAASI,aAAaC,IAAU;IACrC,OAAO,GAAGA,KAAK,KAAK,CAAC,GAAG,EAAEA,KAAK,MAAM,EAAE;AACzC;AAEO,SAASC,gBACdC,QAAoE;IAEpE,MAAMC,cAAc;IACpB,OAAOD,SACJ,GAAG,CAAC,CAACE,OACJ;YACEA,KAAK,EAAE;YACPA,KAAK,IAAI,CAAC,IAAI;YACdA,KAAK,IAAI,CAAC,GAAG;YACbA,KAAK,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI,CAAC,KAAK;YAChCA,KAAK,IAAI,CAAC,GAAG,GAAGA,KAAK,IAAI,CAAC,MAAM;YAChCA,KAAK,OAAO,CAAC,MAAM,GAAGD,cAClB,GAAGC,KAAK,OAAO,CAAC,KAAK,CAAC,GAAGD,aAAa,GAAG,CAAC,GAC1CC,KAAK,OAAO;SACjB,CAAC,IAAI,CAAC,OAER,IAAI,CAAC;AACV;AACO,MAAMC,oBAAoB;AAE1B,SAASC,iCACdC,QAAsC,EACtCC,QAGC,EACDC,OAGC;IAED,MAAMC,wBAAwBD,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,qBAAqB,AAAD,KAAK;IAChE,MAAME,yBAAyBF,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,sBAAsB,AAAD,KAAK;IAElEG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAoB,WAAbJ,UAA0B;IAExC,MAAMK,mBAAkC,EAAE;IAE1C,SAASC,IAAIC,IAAkC;QAC7C,IAAIA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,EAAE;YACd,MAAMX,OAAOW,KAAK,IAAI;YACtB,IACEX,KAAK,IAAI,CAAC,IAAI,IAAII,SAAS,CAAC,IAC5BA,SAAS,CAAC,IAAIJ,KAAK,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI,CAAC,KAAK,IAC9CA,KAAK,IAAI,CAAC,GAAG,IAAII,SAAS,CAAC,IAC3BA,SAAS,CAAC,IAAIJ,KAAK,IAAI,CAAC,GAAG,GAAGA,KAAK,IAAI,CAAC,MAAM,EAC9C;oBAIIY;gBAHJ,IACE,CACEL,CAAAA,0BACAK,AAAAA,SAAAA,CAAAA,mBAAAA,KAAK,UAAU,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,QAAQ,AAAD,MAAMC,0BAAAA,QAAAA,CAAAA,QAAgB,AAAhBA,KAEhCb,KAAK,SAAS,EAEdS,iBAAiB,IAAI,CAACT;YAE1B;QACF;QAEA,KAAK,MAAMc,SAASH,KAAK,QAAQ,CAC/BD,IAAII;IAER;IAEAJ,IAAIP;IAEJ,IAAIM,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB;IAIF,MAAMM,UAAUN,iBAAiB,MAAM,CAAC,CAACO,UAAUC;QACjD,MAAMC,eAAeF,SAAS,IAAI,CAAC,KAAK,GAAGA,SAAS,IAAI,CAAC,MAAM;QAC/D,MAAMG,cAAcF,QAAQ,IAAI,CAAC,KAAK,GAAGA,QAAQ,IAAI,CAAC,MAAM;QAC5D,OAAOE,cAAcD,eAAeD,UAAUD;IAChD;IAEA,MAAMI,mBAAmBC,SACvB;QAAE,GAAGN,QAAQ,MAAM,CAAC,EAAE;QAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;IAAC,GAC7CX;IAGF,IAAIE,uBACF,OAAOc,oBAAoBnB,oBAAoBc,UAAUO;IAG3D,OAAOP;AACT;AAEO,SAASM,SACdE,MAAgC,EAChCC,MAAgC;IAEhC,OAAOC,KAAK,IAAI,CAAEF,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM,IAAKD,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM;AACzE;AAEO,MAAME,wBAAwB,CAAC;;;;;;;;;;;;;;AActC,CAAC;AAEM,eAAeC,iBAGpBC,OAAkD,EAClDC,gBAAmC,EACnCC,GAKC;IAED,MAAM,EAAEC,gBAAgB,EAAE,GAAGH;IAC7B,IAAII;IACJ,IAAIC;IAEJ,IAAIL,QAAQ,IAAI,EACb,GAAEI,KAAK,EAAEC,MAAM,EAAE,GAAGL,QAAQ,IAAG;SAC3B;QACL,MAAMM,UAAU,MAAMC,AAAAA,IAAAA,yBAAAA,iBAAAA,AAAAA,EAAkBJ;QACvC,GAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGC,OAAM;IAC7B;IAEA,MAAM/B,WAAWyB,QAAQ,IAAI;IAE7B,MAAMQ,eAA4C,CAAC;IACnD,MAAMC,eAA8BC,AAAAA,IAAAA,0BAAAA,UAAAA,AAAAA,EAAWnC;IAE/C,IAAI2B,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM,QAAQO,aAAa,MAAM,IAAI,MACtDE,QAAQ,IAAI,CACV;IAIJF,aAAa,OAAO,CAAC,CAACtB;QACpBqB,YAAY,CAACrB,QAAQ,EAAE,CAAC,GAAGA;QAC3B,IAAI,AAA2B,WAApBA,QAAQ,OAAO,EACxBqB,YAAY,CAAC,GAAGrB,QAAQ,OAAO,EAAE,CAAC,GAAGA;IAEzC;IAEA,IAAIyB,kBAAkB;IAEtB,MAAMC,cACJX,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM,iBAAiB,OAAQA,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK;IACpE,MAAMY,mBAAmBZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK,CAACa,AAAAA,IAAAA,oBAAAA,YAAAA,AAAAA,EAAad;IAE3D,IAAIa,kBAAkB;QAEpB,MAAME,cAAc,MAAMC,AAAAA,IAAAA,0BAAAA,iBAAAA,AAAAA,EACxB1C,UACA2B,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,kBAAkB,EACvBA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,oBAAoB,EACzBW;QAIF,MAAMK,kBAAkBnD,aAAa;YAAEqC;YAAOC;QAAO;QACrDO,kBAAkB,CAAC,sBAAsB,EAAEM,gBAAgB,6BAA6B,EAAEF,aAAa;IACzG;IAEA,OAAO;QACL,aAAaJ;QACb,aAAYO,WAAmB;YAC7BvC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAuB,WAAhBuC,aAA6B;YAC3C,MAAM/C,OAAOoC,YAAY,CAAC,GAAGW,aAAa,CAAC;YAC3C,OAAO/C;QACT;QACA,mBACEI,QAAkC,EAClCR,IAAuC;YAEvC,OAAOM,iCAAiCC,UAAUC;QACpD;QACA,yBAAwBA,QAAkC;YACxD,MAAMW,UAAUiC,AAAAA,IAAAA,0BAAAA,yBAAAA,AAAAA,EAA0B5C;YAE1CD,SAAS,QAAQ,CAAC,IAAI,CAAC;gBACrB,MAAMY;gBACN,UAAU,EAAE;YACd;YACAsB,aAAa,IAAI,CAACtB;YAClBqB,YAAY,CAACrB,QAAQ,EAAE,CAAC,GAAGA;YAC3B,OAAOA;QACT;QACA,MAAM;YAAEiB;YAAOC;QAAO;IACxB;AACF"}
+{"version":3,"file":"ai-model/prompt/util.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/prompt/util.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { imageInfoOfBase64 } from '@/image/index';\nimport type { BaseElement, ElementTreeNode, Size, UIContext } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport {\n  descriptionOfTree,\n  generateElementByPosition,\n  treeToList,\n} from '@midscene/shared/extractor';\nimport { assert } from '@midscene/shared/utils';\n\nexport function describeSize(size: Size) {\n  return `${size.width} x ${size.height}`;\n}\n\nexport function describeElement(\n  elements: (Pick<BaseElement, 'rect' | 'content'> & { id: string })[],\n) {\n  const sliceLength = 80;\n  return elements\n    .map((item) =>\n      [\n        item.id,\n        item.rect.left,\n        item.rect.top,\n        item.rect.left + item.rect.width,\n        item.rect.top + item.rect.height,\n        item.content.length > sliceLength\n          ? `${item.content.slice(0, sliceLength)}...`\n          : item.content,\n      ].join(', '),\n    )\n    .join('\\n');\n}\nexport const distanceThreshold = 16;\n\nexport function elementByPositionWithElementInfo(\n  treeRoot: ElementTreeNode<BaseElement>,\n  position: {\n    x: number;\n    y: number;\n  },\n  options?: {\n    requireStrictDistance?: boolean;\n    filterPositionElements?: boolean;\n  },\n) {\n  const requireStrictDistance = options?.requireStrictDistance ?? true;\n  const filterPositionElements = options?.filterPositionElements ?? false;\n\n  assert(typeof position !== 'undefined', 'position is required for query');\n\n  const matchingElements: BaseElement[] = [];\n\n  function dfs(node: ElementTreeNode<BaseElement>) {\n    if (node?.node) {\n      const item = node.node;\n      if (\n        item.rect.left <= position.x &&\n        position.x <= item.rect.left + item.rect.width &&\n        item.rect.top <= position.y &&\n        position.y <= item.rect.top + item.rect.height\n      ) {\n        if (\n          !(\n            filterPositionElements &&\n            item.attributes?.nodeType === NodeType.POSITION\n          ) &&\n          item.isVisible\n        ) {\n          matchingElements.push(item);\n        }\n      }\n    }\n\n    for (const child of node.children) {\n      dfs(child);\n    }\n  }\n\n  dfs(treeRoot);\n\n  if (matchingElements.length === 0) {\n    return undefined;\n  }\n\n  // Find the smallest element by area\n  const element = matchingElements.reduce((smallest, current) => {\n    const smallestArea = smallest.rect.width * smallest.rect.height;\n    const currentArea = current.rect.width * current.rect.height;\n    return currentArea < smallestArea ? current : smallest;\n  });\n\n  const distanceToCenter = distance(\n    { x: element.center[0], y: element.center[1] },\n    position,\n  );\n\n  if (requireStrictDistance) {\n    return distanceToCenter <= distanceThreshold ? element : undefined;\n  }\n\n  return element;\n}\n\nexport function distance(\n  point1: { x: number; y: number },\n  point2: { x: number; y: number },\n) {\n  return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);\n}\n\nexport const samplePageDescription = `\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n`;\n\nexport async function describeUserPage<\n  ElementType extends BaseElement = BaseElement,\n>(\n  context: Omit<UIContext<ElementType>, 'describer'>,\n  opt: {\n    truncateTextLength?: number;\n    filterNonTextContent?: boolean;\n    domIncluded?: boolean | 'visible-only';\n    visibleOnly?: boolean;\n    vlMode: TVlModeTypes | undefined;\n  },\n) {\n  const { screenshotBase64 } = context;\n  let width: number;\n  let height: number;\n\n  if (context.size) {\n    ({ width, height } = context.size);\n  } else {\n    const imgSize = await imageInfoOfBase64(screenshotBase64);\n    ({ width, height } = imgSize);\n  }\n\n  const treeRoot = context.tree;\n  // dfs tree, save the id and element info\n  const idElementMap: Record<string, ElementType> = {};\n  const flatElements: ElementType[] = treeToList(treeRoot);\n\n  if (opt?.domIncluded === true && flatElements.length >= 5000) {\n    console.warn(\n      'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: \"visible-only\" to reduce the number of elements',\n    );\n  }\n\n  flatElements.forEach((element) => {\n    idElementMap[element.id] = element;\n    if (typeof element.indexId !== 'undefined') {\n      idElementMap[`${element.indexId}`] = element;\n    }\n  });\n\n  let pageDescription = '';\n\n  const visibleOnly =\n    opt?.domIncluded === 'visible-only' ? true : (opt?.visibleOnly ?? false);\n  const resolvedVlMode = opt?.vlMode;\n  const shouldIncludeDOM = opt?.domIncluded || !resolvedVlMode;\n\n  if (shouldIncludeDOM) {\n    // non-vl mode must provide the page description\n    const contentTree = await descriptionOfTree(\n      treeRoot,\n      opt?.truncateTextLength,\n      opt?.filterNonTextContent,\n      visibleOnly,\n    );\n\n    // if match by position, don't need to provide the page description\n    const sizeDescription = describeSize({ width, height });\n    pageDescription = `The size of the page: ${sizeDescription} \\n The page elements tree:\\n${contentTree}`;\n  }\n\n  return {\n    description: pageDescription,\n    elementById(idOrIndexId: string) {\n      assert(typeof idOrIndexId !== 'undefined', 'id is required for query');\n      const item = idElementMap[`${idOrIndexId}`];\n      return item;\n    },\n    elementByPosition(\n      position: { x: number; y: number },\n      size: { width: number; height: number },\n    ) {\n      return elementByPositionWithElementInfo(treeRoot, position);\n    },\n    insertElementByPosition(position: { x: number; y: number }) {\n      const element = generateElementByPosition(position) as ElementType;\n\n      treeRoot.children.push({\n        node: element,\n        children: [],\n      });\n      flatElements.push(element);\n      idElementMap[element.id] = element;\n      return element;\n    },\n    size: { width, height },\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","describeSize","size","describeElement","elements","sliceLength","item","distanceThreshold","elementByPositionWithElementInfo","treeRoot","position","options","requireStrictDistance","filterPositionElements","assert","matchingElements","dfs","node","_item_attributes","NodeType","child","element","smallest","current","smallestArea","currentArea","distanceToCenter","distance","undefined","point1","point2","Math","samplePageDescription","describeUserPage","context","opt","screenshotBase64","width","height","imgSize","imageInfoOfBase64","idElementMap","flatElements","treeToList","console","pageDescription","visibleOnly","resolvedVlMode","shouldIncludeDOM","contentTree","descriptionOfTree","sizeDescription","idOrIndexId","generateElementByPosition"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACKO,SAASI,aAAaC,IAAU;IACrC,OAAO,GAAGA,KAAK,KAAK,CAAC,GAAG,EAAEA,KAAK,MAAM,EAAE;AACzC;AAEO,SAASC,gBACdC,QAAoE;IAEpE,MAAMC,cAAc;IACpB,OAAOD,SACJ,GAAG,CAAC,CAACE,OACJ;YACEA,KAAK,EAAE;YACPA,KAAK,IAAI,CAAC,IAAI;YACdA,KAAK,IAAI,CAAC,GAAG;YACbA,KAAK,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI,CAAC,KAAK;YAChCA,KAAK,IAAI,CAAC,GAAG,GAAGA,KAAK,IAAI,CAAC,MAAM;YAChCA,KAAK,OAAO,CAAC,MAAM,GAAGD,cAClB,GAAGC,KAAK,OAAO,CAAC,KAAK,CAAC,GAAGD,aAAa,GAAG,CAAC,GAC1CC,KAAK,OAAO;SACjB,CAAC,IAAI,CAAC,OAER,IAAI,CAAC;AACV;AACO,MAAMC,oBAAoB;AAE1B,SAASC,iCACdC,QAAsC,EACtCC,QAGC,EACDC,OAGC;IAED,MAAMC,wBAAwBD,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,qBAAqB,AAAD,KAAK;IAChE,MAAME,yBAAyBF,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,sBAAsB,AAAD,KAAK;IAElEG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAoB,WAAbJ,UAA0B;IAExC,MAAMK,mBAAkC,EAAE;IAE1C,SAASC,IAAIC,IAAkC;QAC7C,IAAIA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,EAAE;YACd,MAAMX,OAAOW,KAAK,IAAI;YACtB,IACEX,KAAK,IAAI,CAAC,IAAI,IAAII,SAAS,CAAC,IAC5BA,SAAS,CAAC,IAAIJ,KAAK,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI,CAAC,KAAK,IAC9CA,KAAK,IAAI,CAAC,GAAG,IAAII,SAAS,CAAC,IAC3BA,SAAS,CAAC,IAAIJ,KAAK,IAAI,CAAC,GAAG,GAAGA,KAAK,IAAI,CAAC,MAAM,EAC9C;oBAIIY;gBAHJ,IACE,CACEL,CAAAA,0BACAK,AAAAA,SAAAA,CAAAA,mBAAAA,KAAK,UAAU,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,QAAQ,AAAD,MAAMC,0BAAAA,QAAAA,CAAAA,QAAgB,AAAhBA,KAEhCb,KAAK,SAAS,EAEdS,iBAAiB,IAAI,CAACT;YAE1B;QACF;QAEA,KAAK,MAAMc,SAASH,KAAK,QAAQ,CAC/BD,IAAII;IAER;IAEAJ,IAAIP;IAEJ,IAAIM,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB;IAIF,MAAMM,UAAUN,iBAAiB,MAAM,CAAC,CAACO,UAAUC;QACjD,MAAMC,eAAeF,SAAS,IAAI,CAAC,KAAK,GAAGA,SAAS,IAAI,CAAC,MAAM;QAC/D,MAAMG,cAAcF,QAAQ,IAAI,CAAC,KAAK,GAAGA,QAAQ,IAAI,CAAC,MAAM;QAC5D,OAAOE,cAAcD,eAAeD,UAAUD;IAChD;IAEA,MAAMI,mBAAmBC,SACvB;QAAE,GAAGN,QAAQ,MAAM,CAAC,EAAE;QAAE,GAAGA,QAAQ,MAAM,CAAC,EAAE;IAAC,GAC7CX;IAGF,IAAIE,uBACF,OAAOc,oBAAoBnB,oBAAoBc,UAAUO;IAG3D,OAAOP;AACT;AAEO,SAASM,SACdE,MAAgC,EAChCC,MAAgC;IAEhC,OAAOC,KAAK,IAAI,CAAEF,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM,IAAKD,AAAAA,CAAAA,OAAO,CAAC,GAAGC,OAAO,CAAC,AAAD,KAAM;AACzE;AAEO,MAAME,wBAAwB,CAAC;;;;;;;;;;;;;;AActC,CAAC;AAEM,eAAeC,iBAGpBC,OAAkD,EAClDC,GAMC;IAED,MAAM,EAAEC,gBAAgB,EAAE,GAAGF;IAC7B,IAAIG;IACJ,IAAIC;IAEJ,IAAIJ,QAAQ,IAAI,EACb,GAAEG,KAAK,EAAEC,MAAM,EAAE,GAAGJ,QAAQ,IAAG;SAC3B;QACL,MAAMK,UAAU,MAAMC,AAAAA,IAAAA,yBAAAA,iBAAAA,AAAAA,EAAkBJ;QACvC,GAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGC,OAAM;IAC7B;IAEA,MAAM9B,WAAWyB,QAAQ,IAAI;IAE7B,MAAMO,eAA4C,CAAC;IACnD,MAAMC,eAA8BC,AAAAA,IAAAA,0BAAAA,UAAAA,AAAAA,EAAWlC;IAE/C,IAAI0B,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM,QAAQO,aAAa,MAAM,IAAI,MACtDE,QAAQ,IAAI,CACV;IAIJF,aAAa,OAAO,CAAC,CAACrB;QACpBoB,YAAY,CAACpB,QAAQ,EAAE,CAAC,GAAGA;QAC3B,IAAI,AAA2B,WAApBA,QAAQ,OAAO,EACxBoB,YAAY,CAAC,GAAGpB,QAAQ,OAAO,EAAE,CAAC,GAAGA;IAEzC;IAEA,IAAIwB,kBAAkB;IAEtB,MAAMC,cACJX,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM,iBAAiB,OAAQA,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK;IACpE,MAAMY,iBAAiBZ,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,MAAM;IAClC,MAAMa,mBAAmBb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK,CAACY;IAE9C,IAAIC,kBAAkB;QAEpB,MAAMC,cAAc,MAAMC,AAAAA,IAAAA,0BAAAA,iBAAAA,AAAAA,EACxBzC,UACA0B,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,kBAAkB,EACvBA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,oBAAoB,EACzBW;QAIF,MAAMK,kBAAkBlD,aAAa;YAAEoC;YAAOC;QAAO;QACrDO,kBAAkB,CAAC,sBAAsB,EAAEM,gBAAgB,6BAA6B,EAAEF,aAAa;IACzG;IAEA,OAAO;QACL,aAAaJ;QACb,aAAYO,WAAmB;YAC7BtC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAuB,WAAhBsC,aAA6B;YAC3C,MAAM9C,OAAOmC,YAAY,CAAC,GAAGW,aAAa,CAAC;YAC3C,OAAO9C;QACT;QACA,mBACEI,QAAkC,EAClCR,IAAuC;YAEvC,OAAOM,iCAAiCC,UAAUC;QACpD;QACA,yBAAwBA,QAAkC;YACxD,MAAMW,UAAUgC,AAAAA,IAAAA,0BAAAA,yBAAAA,AAAAA,EAA0B3C;YAE1CD,SAAS,QAAQ,CAAC,IAAI,CAAC;gBACrB,MAAMY;gBACN,UAAU,EAAE;YACd;YACAqB,aAAa,IAAI,CAACrB;YAClBoB,YAAY,CAACpB,QAAQ,EAAE,CAAC,GAAGA;YAC3B,OAAOA;QACT;QACA,MAAM;YAAEgB;YAAOC;QAAO;IACxB;AACF"}

package/dist/lib/ai-model/prompt/yaml-generator.js

@@ -129,7 +129,7 @@ const createMessageContent = (promptText, screenshots = [], includeScreenshots =
 const validateEvents = (events)=>{
     if (!events.length) throw new Error('No events provided for test generation');
 };
-const generateYamlTest = async (events, options = {})=>{
+const generateYamlTest = async (events, options, modelConfig)=>{
     try {
         validateEvents(events);
         const summary = prepareEventSummary(events, {
@@ -178,16 +178,14 @@ Respond with YAML only, no explanations.`
                     }))
             });
         }
-        const response = await (0, external_index_js_namespaceObject.callAi)(prompt, external_index_js_namespaceObject.AIActionType.EXTRACT_DATA, {
-            intent: 'default'
-        });
+        const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, external_index_js_namespaceObject.AIActionType.EXTRACT_DATA, modelConfig);
         if ((null == response ? void 0 : response.content) && 'string' == typeof response.content) return response.content;
         throw new Error('Failed to generate YAML test configuration');
     } catch (error) {
         throw new Error(`Failed to generate YAML test: ${error}`);
     }
 };
-const generateYamlTestStream = async (events, options = {})=>{
+const generateYamlTestStream = async (events, options, modelConfig)=>{
     try {
         validateEvents(events);
         const summary = prepareEventSummary(events, {
@@ -236,16 +234,12 @@ Respond with YAML only, no explanations.`
                     }))
             });
         }
-        if (options.stream && options.onChunk) return await (0, external_index_js_namespaceObject.callAi)(prompt, external_index_js_namespaceObject.AIActionType.EXTRACT_DATA, {
-            intent: 'default'
-        }, {
+        if (options.stream && options.onChunk) return await (0, external_index_js_namespaceObject.callAI)(prompt, external_index_js_namespaceObject.AIActionType.EXTRACT_DATA, modelConfig, {
             stream: true,
             onChunk: options.onChunk
         });
         {
-            const response = await (0, external_index_js_namespaceObject.callAi)(prompt, external_index_js_namespaceObject.AIActionType.EXTRACT_DATA, {
-                intent: 'default'
-            });
+            const response = await (0, external_index_js_namespaceObject.callAIWithStringResponse)(prompt, external_index_js_namespaceObject.AIActionType.EXTRACT_DATA, modelConfig);
             if ((null == response ? void 0 : response.content) && 'string' == typeof response.content) return {
                 content: response.content,
                 usage: response.usage,

package/dist/lib/ai-model/prompt/yaml-generator.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/yaml-generator.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/prompt/yaml-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport {\n  AIActionType,\n  type ChatCompletionMessageParam,\n  callAi,\n} from '../index';\n\n// Common interfaces for test generation (shared between YAML and Playwright)\nexport interface EventCounts {\n  navigation: number;\n  click: number;\n  input: number;\n  scroll: number;\n  total: number;\n}\n\nexport interface InputDescription {\n  description: string;\n  value: string;\n}\n\nexport interface ProcessedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n}\n\nexport interface EventSummary {\n  testName: string;\n  startUrl: string;\n  eventCounts: EventCounts;\n  urls: string[];\n  clickDescriptions: string[];\n  inputDescriptions: InputDescription[];\n  events: ProcessedEvent[];\n}\n\n// Common ChromeRecordedEvent interface\nexport interface ChromeRecordedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n  screenshotBefore?: string;\n  screenshotAfter?: string;\n  screenshotWithBox?: string;\n}\n\nexport interface YamlGenerationOptions {\n  testName?: string;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n}\n\nexport interface FilteredEvents {\n  navigationEvents: ChromeRecordedEvent[];\n  clickEvents: ChromeRecordedEvent[];\n  inputEvents: ChromeRecordedEvent[];\n  scrollEvents: ChromeRecordedEvent[];\n}\n\n// Common utility functions (shared between YAML and Playwright generators)\n\n/**\n * Get screenshots from events for LLM context\n */\nexport const getScreenshotsForLLM = (\n  events: ChromeRecordedEvent[],\n  maxScreenshots = 1,\n): string[] => {\n  // Find events with screenshots, prioritizing navigation and click events\n  const eventsWithScreenshots = events.filter(\n    (event) =>\n      event.screenshotBefore ||\n      event.screenshotAfter ||\n      event.screenshotWithBox,\n  );\n\n  // Sort them by priority (navigation first, then clicks, then others)\n  const sortedEvents = [...eventsWithScreenshots].sort((a, b) => {\n    if (a.type === 'navigation' && b.type !== 'navigation') return -1;\n    if (a.type !== 'navigation' && b.type === 'navigation') return 1;\n    if (a.type === 'click' && b.type !== 'click') return -1;\n    if (a.type !== 'click' && b.type === 'click') return 1;\n    return 0;\n  });\n\n  // Extract up to maxScreenshots screenshots\n  const screenshots: string[] = [];\n  for (const event of sortedEvents) {\n    // Prefer the most informative screenshot\n    const screenshot =\n      event.screenshotWithBox ||\n      event.screenshotAfter ||\n      event.screenshotBefore;\n    if (screenshot && !screenshots.includes(screenshot)) {\n      screenshots.push(screenshot);\n      if (screenshots.length >= maxScreenshots) break;\n    }\n  }\n\n  return screenshots;\n};\n\n/**\n * Filter events by type for easier processing\n */\nexport const filterEventsByType = (\n  events: ChromeRecordedEvent[],\n): FilteredEvents => {\n  return {\n    navigationEvents: events.filter((event) => event.type === 'navigation'),\n    clickEvents: events.filter((event) => event.type === 'click'),\n    inputEvents: events.filter((event) => event.type === 'input'),\n    scrollEvents: events.filter((event) => event.type === 'scroll'),\n  };\n};\n\n/**\n * Create event counts summary\n */\nexport const createEventCounts = (\n  filteredEvents: FilteredEvents,\n  totalEvents: number,\n): EventCounts => {\n  return {\n    navigation: filteredEvents.navigationEvents.length,\n    click: filteredEvents.clickEvents.length,\n    input: filteredEvents.inputEvents.length,\n    scroll: filteredEvents.scrollEvents.length,\n    total: totalEvents,\n  };\n};\n\n/**\n * Extract input descriptions from input events\n */\nexport const extractInputDescriptions = (\n  inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n  return inputEvents\n    .map((event) => ({\n      description: event.elementDescription || '',\n      value: event.value || '',\n    }))\n    .filter((item) => item.description && item.value);\n};\n\n/**\n * Process events for LLM consumption\n */\nexport const processEventsForLLM = (\n  events: ChromeRecordedEvent[],\n): ProcessedEvent[] => {\n  return events.map((event) => ({\n    type: event.type,\n    timestamp: event.timestamp,\n    url: event.url,\n    title: event.title,\n    elementDescription: event.elementDescription,\n    value: event.value,\n    pageInfo: event.pageInfo,\n    elementRect: event.elementRect,\n  }));\n};\n\n/**\n * Prepare comprehensive event summary for LLM\n */\nexport const prepareEventSummary = (\n  events: ChromeRecordedEvent[],\n  options: { testName?: string; maxScreenshots?: number } = {},\n): EventSummary => {\n  const filteredEvents = filterEventsByType(events);\n  const eventCounts = createEventCounts(filteredEvents, events.length);\n\n  // Extract useful information from events\n  const startUrl =\n    filteredEvents.navigationEvents.length > 0\n      ? filteredEvents.navigationEvents[0].url || ''\n      : '';\n\n  const clickDescriptions = filteredEvents.clickEvents\n    .map((event) => event.elementDescription)\n    .filter((desc): desc is string => Boolean(desc))\n    .slice(0, 10);\n\n  const inputDescriptions = extractInputDescriptions(\n    filteredEvents.inputEvents,\n  ).slice(0, 10);\n\n  const urls = filteredEvents.navigationEvents\n    .map((e) => e.url)\n    .filter((url): url is string => Boolean(url))\n    .slice(0, 5);\n\n  const processedEvents = processEventsForLLM(events);\n\n  return {\n    testName: options.testName || 'Automated test from recorded events',\n    startUrl,\n    eventCounts,\n    urls,\n    clickDescriptions,\n    inputDescriptions,\n    events: processedEvents,\n  };\n};\n\n/**\n * Create message content for LLM with optional screenshots\n */\nexport const createMessageContent = (\n  promptText: string,\n  screenshots: string[] = [],\n  includeScreenshots = true,\n) => {\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  // Add screenshots if available and requested\n  if (includeScreenshots && screenshots.length > 0) {\n    messageContent.unshift({\n      type: 'text',\n      text: 'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    screenshots.forEach((screenshot) => {\n      messageContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      });\n    });\n  }\n\n  return messageContent;\n};\n\n/**\n * Validate events before processing\n */\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n  if (!events.length) {\n    throw new Error('No events provided for test generation');\n  }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions = {},\n): Promise<string> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    // Use LLM to generate the YAML test configuration\n    const prompt: ChatCompletionMessageParam[] = [\n      {\n        role: 'system',\n        content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n      },\n      {\n        role: 'user',\n        content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nRespond with YAML only, no explanations.`,\n      },\n    ];\n\n    // Add screenshots if available and requested\n    if (screenshots.length > 0) {\n      prompt.push({\n        role: 'user',\n        content:\n          'Here are screenshots from the recording session to help you understand the context:',\n      });\n\n      prompt.push({\n        role: 'user',\n        content: screenshots.map((screenshot) => ({\n          type: 'image_url',\n          image_url: {\n            url: screenshot,\n          },\n        })),\n      });\n    }\n\n    const response = await callAi(prompt, AIActionType.EXTRACT_DATA, {\n      intent: 'default',\n    });\n\n    if (response?.content && typeof response.content === 'string') {\n      return response.content;\n    }\n\n    throw new Error('Failed to generate YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions & StreamingCodeGenerationOptions = {},\n): Promise<StreamingAIResponse> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    // Use LLM to generate the YAML test configuration\n    const prompt: ChatCompletionMessageParam[] = [\n      {\n        role: 'system',\n        content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n      },\n      {\n        role: 'user',\n        content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nRespond with YAML only, no explanations.`,\n      },\n    ];\n\n    // Add screenshots if available and requested\n    if (screenshots.length > 0) {\n      prompt.push({\n        role: 'user',\n        content:\n          'Here are screenshots from the recording session to help you understand the context:',\n      });\n\n      prompt.push({\n        role: 'user',\n        content: screenshots.map((screenshot) => ({\n          type: 'image_url',\n          image_url: {\n            url: screenshot,\n          },\n        })),\n      });\n    }\n\n    if (options.stream && options.onChunk) {\n      // Use streaming\n      return await callAi(\n        prompt,\n        AIActionType.EXTRACT_DATA,\n        {\n          intent: 'default',\n        },\n        {\n          stream: true,\n          onChunk: options.onChunk,\n        },\n      );\n    } else {\n      // Fallback to non-streaming\n      const response = await callAi(prompt, AIActionType.EXTRACT_DATA, {\n        intent: 'default',\n      });\n\n      if (response?.content && typeof response.content === 'string') {\n        return {\n          content: response.content,\n          usage: response.usage,\n          isStreamed: false,\n        };\n      }\n\n      throw new Error('Failed to generate YAML test configuration');\n    }\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getScreenshotsForLLM","events","maxScreenshots","eventsWithScreenshots","event","sortedEvents","a","b","screenshots","screenshot","filterEventsByType","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","item","processEventsForLLM","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","Boolean","inputDescriptions","urls","e","url","processedEvents","createMessageContent","promptText","includeScreenshots","messageContent","validateEvents","Error","generateYamlTest","summary","yamlSummary","prompt","YAML_EXAMPLE_CODE","JSON","response","callAi","AIActionType","error","generateYamlTestStream"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;AC0EO,MAAMI,uBAAuB,CAClCC,QACAC,iBAAiB,CAAC;IAGlB,MAAMC,wBAAwBF,OAAO,MAAM,CACzC,CAACG,QACCA,MAAM,gBAAgB,IACtBA,MAAM,eAAe,IACrBA,MAAM,iBAAiB;IAI3B,MAAMC,eAAe;WAAIF;KAAsB,CAAC,IAAI,CAAC,CAACG,GAAGC;QACvD,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,OAAO;IACT;IAGA,MAAMC,cAAwB,EAAE;IAChC,KAAK,MAAMJ,SAASC,aAAc;QAEhC,MAAMI,aACJL,MAAM,iBAAiB,IACvBA,MAAM,eAAe,IACrBA,MAAM,gBAAgB;QACxB,IAAIK,cAAc,CAACD,YAAY,QAAQ,CAACC,aAAa;YACnDD,YAAY,IAAI,CAACC;YACjB,IAAID,YAAY,MAAM,IAAIN,gBAAgB;QAC5C;IACF;IAEA,OAAOM;AACT;AAKO,MAAME,qBAAqB,CAChCT,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAMK,MAAMO,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAMK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACX,QAAW;YACf,aAAaA,MAAM,kBAAkB,IAAI;YACzC,OAAOA,MAAM,KAAK,IAAI;QACxB,IACC,MAAM,CAAC,CAACY,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAM7C,MAAMC,sBAAsB,CACjChB,SAEOA,OAAO,GAAG,CAAC,CAACG,QAAW;YAC5B,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClB,oBAAoBA,MAAM,kBAAkB;YAC5C,OAAOA,MAAM,KAAK;YAClB,UAAUA,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;QAChC;AAMK,MAAMc,sBAAsB,CACjCjB,QACAkB,UAA0D,CAAC,CAAC;IAE5D,MAAMP,iBAAiBF,mBAAmBT;IAC1C,MAAMmB,cAAcT,kBAAkBC,gBAAgBX,OAAO,MAAM;IAGnE,MAAMoB,WACJT,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAMU,oBAAoBV,eAAe,WAAW,CACjD,GAAG,CAAC,CAACR,QAAUA,MAAM,kBAAkB,EACvC,MAAM,CAAC,CAACmB,OAAyBC,QAAQD,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAME,oBAAoBX,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAMc,OAAOd,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAACe,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBJ,QAAQI,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkBZ,oBAAoBhB;IAE5C,OAAO;QACL,UAAUkB,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAM;QACAJ;QACAG;QACA,QAAQI;IACV;AACF;AAKO,MAAMC,uBAAuB,CAClCC,YACAvB,cAAwB,EAAE,EAC1BwB,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMF;QACR;KACD;IAGD,IAAIC,sBAAsBxB,YAAY,MAAM,GAAG,GAAG;QAChDyB,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAzB,YAAY,OAAO,CAAC,CAACC;YACnBwB,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKxB;gBACP;YACF;QACF;IACF;IAEA,OAAOwB;AACT;AAKO,MAAMC,iBAAiB,CAACjC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIkC,MAAM;AAEpB;AAOO,MAAMC,mBAAmB,OAC9BnC,QACAkB,UAAiC,CAAC,CAAC;IAEnC,IAAI;QAEFe,eAAejC;QAGf,MAAMoC,UAAUnB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMmB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBlB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMoB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;wCASC,CAAC;YACnC;SACD;QAGD,IAAI9B,YAAY,MAAM,GAAG,GAAG;YAC1B+B,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAS/B,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,MAAMiC,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOJ,QAAQK,kCAAAA,YAAAA,CAAAA,YAAyB,EAAE;YAC/D,QAAQ;QACV;QAEA,IAAIF,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;QAGzB,MAAM,IAAIP,MAAM;IAClB,EAAE,OAAOU,OAAO;QACd,MAAM,IAAIV,MAAM,CAAC,8BAA8B,EAAEU,OAAO;IAC1D;AACF;AAKO,MAAMC,yBAAyB,OACpC7C,QACAkB,UAAkE,CAAC,CAAC;IAEpE,IAAI;QAEFe,eAAejC;QAGf,MAAMoC,UAAUnB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMmB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBlB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMoB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;wCASC,CAAC;YACnC;SACD;QAGD,IAAI9B,YAAY,MAAM,GAAG,GAAG;YAC1B+B,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAS/B,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,IAAIU,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMwB,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EACXJ,QACAK,kCAAAA,YAAAA,CAAAA,YAAyB,EACzB;YACE,QAAQ;QACV,GACA;YACE,QAAQ;YACR,SAASzB,QAAQ,OAAO;QAC1B;QAEG;YAEL,MAAMuB,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOJ,QAAQK,kCAAAA,YAAAA,CAAAA,YAAyB,EAAE;gBAC/D,QAAQ;YACV;YAEA,IAAIF,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;gBACL,SAASA,SAAS,OAAO;gBACzB,OAAOA,SAAS,KAAK;gBACrB,YAAY;YACd;YAGF,MAAM,IAAIP,MAAM;QAClB;IACF,EAAE,OAAOU,OAAO;QACd,MAAM,IAAIV,MAAM,CAAC,8BAA8B,EAAEU,OAAO;IAC1D;AACF"}
+{"version":3,"file":"ai-model/prompt/yaml-generator.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/prompt/yaml-generator.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  AIActionType,\n  type ChatCompletionMessageParam,\n  callAI,\n  callAIWithStringResponse,\n} from '../index';\n\n// Common interfaces for test generation (shared between YAML and Playwright)\nexport interface EventCounts {\n  navigation: number;\n  click: number;\n  input: number;\n  scroll: number;\n  total: number;\n}\n\nexport interface InputDescription {\n  description: string;\n  value: string;\n}\n\nexport interface ProcessedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n}\n\nexport interface EventSummary {\n  testName: string;\n  startUrl: string;\n  eventCounts: EventCounts;\n  urls: string[];\n  clickDescriptions: string[];\n  inputDescriptions: InputDescription[];\n  events: ProcessedEvent[];\n}\n\n// Common ChromeRecordedEvent interface\nexport interface ChromeRecordedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n  screenshotBefore?: string;\n  screenshotAfter?: string;\n  screenshotWithBox?: string;\n}\n\nexport interface YamlGenerationOptions {\n  testName?: string;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n}\n\nexport interface FilteredEvents {\n  navigationEvents: ChromeRecordedEvent[];\n  clickEvents: ChromeRecordedEvent[];\n  inputEvents: ChromeRecordedEvent[];\n  scrollEvents: ChromeRecordedEvent[];\n}\n\n// Common utility functions (shared between YAML and Playwright generators)\n\n/**\n * Get screenshots from events for LLM context\n */\nexport const getScreenshotsForLLM = (\n  events: ChromeRecordedEvent[],\n  maxScreenshots = 1,\n): string[] => {\n  // Find events with screenshots, prioritizing navigation and click events\n  const eventsWithScreenshots = events.filter(\n    (event) =>\n      event.screenshotBefore ||\n      event.screenshotAfter ||\n      event.screenshotWithBox,\n  );\n\n  // Sort them by priority (navigation first, then clicks, then others)\n  const sortedEvents = [...eventsWithScreenshots].sort((a, b) => {\n    if (a.type === 'navigation' && b.type !== 'navigation') return -1;\n    if (a.type !== 'navigation' && b.type === 'navigation') return 1;\n    if (a.type === 'click' && b.type !== 'click') return -1;\n    if (a.type !== 'click' && b.type === 'click') return 1;\n    return 0;\n  });\n\n  // Extract up to maxScreenshots screenshots\n  const screenshots: string[] = [];\n  for (const event of sortedEvents) {\n    // Prefer the most informative screenshot\n    const screenshot =\n      event.screenshotWithBox ||\n      event.screenshotAfter ||\n      event.screenshotBefore;\n    if (screenshot && !screenshots.includes(screenshot)) {\n      screenshots.push(screenshot);\n      if (screenshots.length >= maxScreenshots) break;\n    }\n  }\n\n  return screenshots;\n};\n\n/**\n * Filter events by type for easier processing\n */\nexport const filterEventsByType = (\n  events: ChromeRecordedEvent[],\n): FilteredEvents => {\n  return {\n    navigationEvents: events.filter((event) => event.type === 'navigation'),\n    clickEvents: events.filter((event) => event.type === 'click'),\n    inputEvents: events.filter((event) => event.type === 'input'),\n    scrollEvents: events.filter((event) => event.type === 'scroll'),\n  };\n};\n\n/**\n * Create event counts summary\n */\nexport const createEventCounts = (\n  filteredEvents: FilteredEvents,\n  totalEvents: number,\n): EventCounts => {\n  return {\n    navigation: filteredEvents.navigationEvents.length,\n    click: filteredEvents.clickEvents.length,\n    input: filteredEvents.inputEvents.length,\n    scroll: filteredEvents.scrollEvents.length,\n    total: totalEvents,\n  };\n};\n\n/**\n * Extract input descriptions from input events\n */\nexport const extractInputDescriptions = (\n  inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n  return inputEvents\n    .map((event) => ({\n      description: event.elementDescription || '',\n      value: event.value || '',\n    }))\n    .filter((item) => item.description && item.value);\n};\n\n/**\n * Process events for LLM consumption\n */\nexport const processEventsForLLM = (\n  events: ChromeRecordedEvent[],\n): ProcessedEvent[] => {\n  return events.map((event) => ({\n    type: event.type,\n    timestamp: event.timestamp,\n    url: event.url,\n    title: event.title,\n    elementDescription: event.elementDescription,\n    value: event.value,\n    pageInfo: event.pageInfo,\n    elementRect: event.elementRect,\n  }));\n};\n\n/**\n * Prepare comprehensive event summary for LLM\n */\nexport const prepareEventSummary = (\n  events: ChromeRecordedEvent[],\n  options: { testName?: string; maxScreenshots?: number } = {},\n): EventSummary => {\n  const filteredEvents = filterEventsByType(events);\n  const eventCounts = createEventCounts(filteredEvents, events.length);\n\n  // Extract useful information from events\n  const startUrl =\n    filteredEvents.navigationEvents.length > 0\n      ? filteredEvents.navigationEvents[0].url || ''\n      : '';\n\n  const clickDescriptions = filteredEvents.clickEvents\n    .map((event) => event.elementDescription)\n    .filter((desc): desc is string => Boolean(desc))\n    .slice(0, 10);\n\n  const inputDescriptions = extractInputDescriptions(\n    filteredEvents.inputEvents,\n  ).slice(0, 10);\n\n  const urls = filteredEvents.navigationEvents\n    .map((e) => e.url)\n    .filter((url): url is string => Boolean(url))\n    .slice(0, 5);\n\n  const processedEvents = processEventsForLLM(events);\n\n  return {\n    testName: options.testName || 'Automated test from recorded events',\n    startUrl,\n    eventCounts,\n    urls,\n    clickDescriptions,\n    inputDescriptions,\n    events: processedEvents,\n  };\n};\n\n/**\n * Create message content for LLM with optional screenshots\n */\nexport const createMessageContent = (\n  promptText: string,\n  screenshots: string[] = [],\n  includeScreenshots = true,\n) => {\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  // Add screenshots if available and requested\n  if (includeScreenshots && screenshots.length > 0) {\n    messageContent.unshift({\n      type: 'text',\n      text: 'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    screenshots.forEach((screenshot) => {\n      messageContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      });\n    });\n  }\n\n  return messageContent;\n};\n\n/**\n * Validate events before processing\n */\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n  if (!events.length) {\n    throw new Error('No events provided for test generation');\n  }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    // Use LLM to generate the YAML test configuration\n    const prompt: ChatCompletionMessageParam[] = [\n      {\n        role: 'system',\n        content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n      },\n      {\n        role: 'user',\n        content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nRespond with YAML only, no explanations.`,\n      },\n    ];\n\n    // Add screenshots if available and requested\n    if (screenshots.length > 0) {\n      prompt.push({\n        role: 'user',\n        content:\n          'Here are screenshots from the recording session to help you understand the context:',\n      });\n\n      prompt.push({\n        role: 'user',\n        content: screenshots.map((screenshot) => ({\n          type: 'image_url',\n          image_url: {\n            url: screenshot,\n          },\n        })),\n      });\n    }\n\n    const response = await callAIWithStringResponse(\n      prompt,\n      AIActionType.EXTRACT_DATA,\n      modelConfig,\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return response.content;\n    }\n\n    throw new Error('Failed to generate YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    // Use LLM to generate the YAML test configuration\n    const prompt: ChatCompletionMessageParam[] = [\n      {\n        role: 'system',\n        content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n      },\n      {\n        role: 'user',\n        content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nRespond with YAML only, no explanations.`,\n      },\n    ];\n\n    // Add screenshots if available and requested\n    if (screenshots.length > 0) {\n      prompt.push({\n        role: 'user',\n        content:\n          'Here are screenshots from the recording session to help you understand the context:',\n      });\n\n      prompt.push({\n        role: 'user',\n        content: screenshots.map((screenshot) => ({\n          type: 'image_url',\n          image_url: {\n            url: screenshot,\n          },\n        })),\n      });\n    }\n\n    if (options.stream && options.onChunk) {\n      // Use streaming\n      return await callAI(prompt, AIActionType.EXTRACT_DATA, modelConfig, {\n        stream: true,\n        onChunk: options.onChunk,\n      });\n    } else {\n      // Fallback to non-streaming\n      const response = await callAIWithStringResponse(\n        prompt,\n        AIActionType.EXTRACT_DATA,\n        modelConfig,\n      );\n\n      if (response?.content && typeof response.content === 'string') {\n        return {\n          content: response.content,\n          usage: response.usage,\n          isStreamed: false,\n        };\n      }\n\n      throw new Error('Failed to generate YAML test configuration');\n    }\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","getScreenshotsForLLM","events","maxScreenshots","eventsWithScreenshots","event","sortedEvents","a","b","screenshots","screenshot","filterEventsByType","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","item","processEventsForLLM","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","Boolean","inputDescriptions","urls","e","url","processedEvents","createMessageContent","promptText","includeScreenshots","messageContent","validateEvents","Error","generateYamlTest","modelConfig","summary","yamlSummary","prompt","YAML_EXAMPLE_CODE","JSON","response","callAIWithStringResponse","AIActionType","error","generateYamlTestStream","callAI"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;AC4EO,MAAMI,uBAAuB,CAClCC,QACAC,iBAAiB,CAAC;IAGlB,MAAMC,wBAAwBF,OAAO,MAAM,CACzC,CAACG,QACCA,MAAM,gBAAgB,IACtBA,MAAM,eAAe,IACrBA,MAAM,iBAAiB;IAI3B,MAAMC,eAAe;WAAIF;KAAsB,CAAC,IAAI,CAAC,CAACG,GAAGC;QACvD,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,OAAO;IACT;IAGA,MAAMC,cAAwB,EAAE;IAChC,KAAK,MAAMJ,SAASC,aAAc;QAEhC,MAAMI,aACJL,MAAM,iBAAiB,IACvBA,MAAM,eAAe,IACrBA,MAAM,gBAAgB;QACxB,IAAIK,cAAc,CAACD,YAAY,QAAQ,CAACC,aAAa;YACnDD,YAAY,IAAI,CAACC;YACjB,IAAID,YAAY,MAAM,IAAIN,gBAAgB;QAC5C;IACF;IAEA,OAAOM;AACT;AAKO,MAAME,qBAAqB,CAChCT,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAMK,MAAMO,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAMK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACX,QAAW;YACf,aAAaA,MAAM,kBAAkB,IAAI;YACzC,OAAOA,MAAM,KAAK,IAAI;QACxB,IACC,MAAM,CAAC,CAACY,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAM7C,MAAMC,sBAAsB,CACjChB,SAEOA,OAAO,GAAG,CAAC,CAACG,QAAW;YAC5B,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClB,oBAAoBA,MAAM,kBAAkB;YAC5C,OAAOA,MAAM,KAAK;YAClB,UAAUA,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;QAChC;AAMK,MAAMc,sBAAsB,CACjCjB,QACAkB,UAA0D,CAAC,CAAC;IAE5D,MAAMP,iBAAiBF,mBAAmBT;IAC1C,MAAMmB,cAAcT,kBAAkBC,gBAAgBX,OAAO,MAAM;IAGnE,MAAMoB,WACJT,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAMU,oBAAoBV,eAAe,WAAW,CACjD,GAAG,CAAC,CAACR,QAAUA,MAAM,kBAAkB,EACvC,MAAM,CAAC,CAACmB,OAAyBC,QAAQD,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAME,oBAAoBX,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAMc,OAAOd,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAACe,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBJ,QAAQI,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkBZ,oBAAoBhB;IAE5C,OAAO;QACL,UAAUkB,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAM;QACAJ;QACAG;QACA,QAAQI;IACV;AACF;AAKO,MAAMC,uBAAuB,CAClCC,YACAvB,cAAwB,EAAE,EAC1BwB,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMF;QACR;KACD;IAGD,IAAIC,sBAAsBxB,YAAY,MAAM,GAAG,GAAG;QAChDyB,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAzB,YAAY,OAAO,CAAC,CAACC;YACnBwB,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKxB;gBACP;YACF;QACF;IACF;IAEA,OAAOwB;AACT;AAKO,MAAMC,iBAAiB,CAACjC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIkC,MAAM;AAEpB;AAOO,MAAMC,mBAAmB,OAC9BnC,QACAkB,SACAkB;IAEA,IAAI;QAEFH,eAAejC;QAGf,MAAMqC,UAAUpB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMoB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBnB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMqB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;wCASC,CAAC;YACnC;SACD;QAGD,IAAI/B,YAAY,MAAM,GAAG,GAAG;YAC1BgC,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAShC,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,MAAMkC,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACrBJ,QACAK,kCAAAA,YAAAA,CAAAA,YAAyB,EACzBR;QAGF,IAAIM,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;QAGzB,MAAM,IAAIR,MAAM;IAClB,EAAE,OAAOW,OAAO;QACd,MAAM,IAAIX,MAAM,CAAC,8BAA8B,EAAEW,OAAO;IAC1D;AACF;AAKO,MAAMC,yBAAyB,OACpC9C,QACAkB,SACAkB;IAEA,IAAI;QAEFH,eAAejC;QAGf,MAAMqC,UAAUpB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMoB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBnB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMqB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,0BAAAA,iBAAiBA,EAAE;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;wCASC,CAAC;YACnC;SACD;QAGD,IAAI/B,YAAY,MAAM,GAAG,GAAG;YAC1BgC,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAShC,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,IAAIU,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAM6B,AAAAA,IAAAA,kCAAAA,MAAAA,AAAAA,EAAOR,QAAQK,kCAAAA,YAAAA,CAAAA,YAAyB,EAAER,aAAa;YAClE,QAAQ;YACR,SAASlB,QAAQ,OAAO;QAC1B;QACK;YAEL,MAAMwB,WAAW,MAAMC,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACrBJ,QACAK,kCAAAA,YAAAA,CAAAA,YAAyB,EACzBR;YAGF,IAAIM,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;gBACL,SAASA,SAAS,OAAO;gBACzB,OAAOA,SAAS,KAAK;gBACrB,YAAY;YACd;YAGF,MAAM,IAAIR,MAAM;QAClB;IACF,EAAE,OAAOW,OAAO;QACd,MAAM,IAAIX,MAAM,CAAC,8BAA8B,EAAEW,OAAO;IAC1D;AACF"}

package/dist/lib/ai-model/service-caller/index.js

@@ -51,12 +51,12 @@ var __webpack_exports__ = {};
     __webpack_require__.r(__webpack_exports__);
     __webpack_require__.d(__webpack_exports__, {
         extractJSONFromCodeBlock: ()=>extractJSONFromCodeBlock,
+        callAIWithStringResponse: ()=>callAIWithStringResponse,
         preprocessDoubaoBboxJson: ()=>preprocessDoubaoBboxJson,
-        callToGetJSONObject: ()=>callToGetJSONObject,
-        callAiFnWithStringResponse: ()=>callAiFnWithStringResponse,
+        callAIWithObjectResponse: ()=>callAIWithObjectResponse,
         getResponseFormat: ()=>getResponseFormat,
-        call: ()=>call,
-        safeParseJson: ()=>safeParseJson
+        safeParseJson: ()=>safeParseJson,
+        callAI: ()=>callAI
     });
     const external_types_js_namespaceObject = require("../../types.js");
     const sdk_namespaceObject = require("@anthropic-ai/sdk");
@@ -74,8 +74,8 @@ var __webpack_exports__ = {};
     const assertion_js_namespaceObject = require("../prompt/assertion.js");
     const llm_locator_js_namespaceObject = require("../prompt/llm-locator.js");
     const llm_planning_js_namespaceObject = require("../prompt/llm-planning.js");
-    async function createChatClient({ AIActionTypeValue, modelPreferences }) {
-        const { socksProxy, httpProxy, modelName, openaiBaseURL, openaiApiKey, openaiExtraConfig, openaiUseAzureDeprecated, useAzureOpenai, azureOpenaiScope, azureOpenaiKey, azureOpenaiEndpoint, azureOpenaiApiVersion, azureOpenaiDeployment, azureExtraConfig, useAnthropicSdk, anthropicApiKey, modelDescription } = env_namespaceObject.globalConfigManager.getModelConfigByIntent(modelPreferences.intent);
+    async function createChatClient({ AIActionTypeValue, modelConfig }) {
+        const { socksProxy, httpProxy, modelName, openaiBaseURL, openaiApiKey, openaiExtraConfig, openaiUseAzureDeprecated, useAzureOpenai, azureOpenaiScope, azureOpenaiKey, azureOpenaiEndpoint, azureOpenaiApiVersion, azureOpenaiDeployment, azureExtraConfig, useAnthropicSdk, anthropicApiKey, modelDescription, uiTarsModelVersion: uiTarsVersion, vlMode } = modelConfig;
         let openai;
         let proxyAgent;
         const debugProxy = (0, logger_namespaceObject.getDebug)('ai:call:proxy');
@@ -137,7 +137,9 @@ var __webpack_exports__ = {};
             completion: openai.chat.completions,
             style: 'openai',
             modelName,
-            modelDescription
+            modelDescription,
+            uiTarsVersion,
+            vlMode
         };
         if (useAnthropicSdk) openai = new sdk_namespaceObject.Anthropic({
             apiKey: anthropicApiKey,
@@ -148,14 +150,16 @@ var __webpack_exports__ = {};
             completion: openai.messages,
             style: 'anthropic',
             modelName,
-            modelDescription
+            modelDescription,
+            uiTarsVersion,
+            vlMode
         };
         throw new Error('Openai SDK or Anthropic SDK is not initialized');
     }
-    async function call(messages, AIActionTypeValue, modelPreferences, options) {
-        const { completion, style, modelName, modelDescription } = await createChatClient({
+    async function callAI(messages, AIActionTypeValue, modelConfig, options) {
+        const { completion, style, modelName, modelDescription, uiTarsVersion, vlMode } = await createChatClient({
             AIActionTypeValue,
-            modelPreferences
+            modelConfig
         });
         const responseFormat = getResponseFormat(modelName, AIActionTypeValue);
         const maxTokens = env_namespaceObject.globalConfigManager.getEnvConfigValue(env_namespaceObject.OPENAI_MAX_TOKENS);
@@ -169,10 +173,10 @@ var __webpack_exports__ = {};
         let usage;
         let timeCost;
         const commonConfig = {
-            temperature: 'vlm-ui-tars' === (0, env_namespaceObject.vlLocateMode)(modelPreferences) ? 0.0 : 0.1,
+            temperature: 'vlm-ui-tars' === vlMode ? 0.0 : 0.1,
             stream: !!isStreaming,
             max_tokens: 'number' == typeof maxTokens ? maxTokens : Number.parseInt(maxTokens || '2048', 10),
-            ...'qwen-vl' === (0, env_namespaceObject.vlLocateMode)(modelPreferences) ? {
+            ...'qwen-vl' === vlMode ? {
                 vl_high_resolution_images: true
             } : {}
         };
@@ -226,7 +230,7 @@ var __webpack_exports__ = {};
                                     time_cost: timeCost ?? 0,
                                     model_name: modelName,
                                     model_description: modelDescription,
-                                    intent: modelPreferences.intent
+                                    intent: modelConfig.intent
                                 }
                             };
                             options.onChunk(finalChunk);
@@ -234,7 +238,7 @@ var __webpack_exports__ = {};
                         }
                     }
                     content = accumulated;
-                    debugProfileStats(`streaming model, ${modelName}, mode, ${(0, env_namespaceObject.vlLocateMode)(modelPreferences) || 'default'}, cost-ms, ${timeCost}`);
+                    debugProfileStats(`streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`);
                 } else {
                     var _result_usage, _result_usage1, _result_usage2;
                     const result = await completion.create({
@@ -244,7 +248,7 @@ var __webpack_exports__ = {};
                         ...commonConfig
                     });
                     timeCost = Date.now() - startTime;
-                    debugProfileStats(`model, ${modelName}, mode, ${(0, env_namespaceObject.vlLocateMode)(modelPreferences) || 'default'}, ui-tars-version, ${(0, env_namespaceObject.uiTarsModelVersion)(modelPreferences)}, prompt-tokens, ${(null == (_result_usage = result.usage) ? void 0 : _result_usage.prompt_tokens) || ''}, completion-tokens, ${(null == (_result_usage1 = result.usage) ? void 0 : _result_usage1.completion_tokens) || ''}, total-tokens, ${(null == (_result_usage2 = result.usage) ? void 0 : _result_usage2.total_tokens) || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`);
+                    debugProfileStats(`model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${(null == (_result_usage = result.usage) ? void 0 : _result_usage.prompt_tokens) || ''}, completion-tokens, ${(null == (_result_usage1 = result.usage) ? void 0 : _result_usage1.completion_tokens) || ''}, total-tokens, ${(null == (_result_usage2 = result.usage) ? void 0 : _result_usage2.total_tokens) || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`);
                     debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);
                     (0, utils_namespaceObject.assert)(result.choices, `invalid response from LLM service: ${JSON.stringify(result)}`);
                     content = result.choices[0].message.content;
@@ -309,7 +313,7 @@ var __webpack_exports__ = {};
                                     time_cost: timeCost ?? 0,
                                     model_name: modelName,
                                     model_description: modelDescription,
-                                    intent: modelPreferences.intent
+                                    intent: modelConfig.intent
                                 } : void 0
                             };
                             options.onChunk(finalChunk);
@@ -351,7 +355,7 @@ var __webpack_exports__ = {};
                     time_cost: timeCost ?? 0,
                     model_name: modelName,
                     model_description: modelDescription,
-                    intent: modelPreferences.intent
+                    intent: modelConfig.intent
                 } : void 0,
                 isStreamed: !!isStreaming
             };
@@ -387,17 +391,18 @@ var __webpack_exports__ = {};
         };
         return responseFormat;
     };
-    async function callToGetJSONObject(messages, AIActionTypeValue, modelPreferences) {
-        const response = await call(messages, AIActionTypeValue, modelPreferences);
+    async function callAIWithObjectResponse(messages, AIActionTypeValue, modelConfig) {
+        const response = await callAI(messages, AIActionTypeValue, modelConfig);
         (0, utils_namespaceObject.assert)(response, 'empty response');
-        const jsonContent = safeParseJson(response.content, modelPreferences);
+        const vlMode = modelConfig.vlMode;
+        const jsonContent = safeParseJson(response.content, vlMode);
         return {
             content: jsonContent,
             usage: response.usage
         };
     }
-    async function callAiFnWithStringResponse(msgs, AIActionTypeValue, modelPreferences) {
-        const { content, usage } = await call(msgs, AIActionTypeValue, modelPreferences);
+    async function callAIWithStringResponse(msgs, AIActionTypeValue, modelConfig) {
+        const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);
         return {
             content,
             usage
@@ -418,7 +423,7 @@ var __webpack_exports__ = {};
         if (input.includes('bbox')) while(/\d+\s+\d+/.test(input))input = input.replace(/(\d+)\s+(\d+)/g, '$1,$2');
         return input;
     }
-    function safeParseJson(input, modelPreferences) {
+    function safeParseJson(input, vlMode) {
         const cleanJsonString = extractJSONFromCodeBlock(input);
         if (null == cleanJsonString ? void 0 : cleanJsonString.match(/\((\d+),(\d+)\)/)) {
             var _cleanJsonString_match;
@@ -430,24 +435,24 @@ var __webpack_exports__ = {};
         try {
             return JSON.parse((0, external_jsonrepair_namespaceObject.jsonrepair)(cleanJsonString));
         } catch (e) {}
-        if ('doubao-vision' === (0, env_namespaceObject.vlLocateMode)(modelPreferences) || 'vlm-ui-tars' === (0, env_namespaceObject.vlLocateMode)(modelPreferences)) {
+        if ('doubao-vision' === vlMode || 'vlm-ui-tars' === vlMode) {
             const jsonString = preprocessDoubaoBboxJson(cleanJsonString);
             return JSON.parse((0, external_jsonrepair_namespaceObject.jsonrepair)(jsonString));
         }
         throw Error(`failed to parse json response: ${input}`);
     }
 })();
-exports.call = __webpack_exports__.call;
-exports.callAiFnWithStringResponse = __webpack_exports__.callAiFnWithStringResponse;
-exports.callToGetJSONObject = __webpack_exports__.callToGetJSONObject;
+exports.callAI = __webpack_exports__.callAI;
+exports.callAIWithObjectResponse = __webpack_exports__.callAIWithObjectResponse;
+exports.callAIWithStringResponse = __webpack_exports__.callAIWithStringResponse;
 exports.extractJSONFromCodeBlock = __webpack_exports__.extractJSONFromCodeBlock;
 exports.getResponseFormat = __webpack_exports__.getResponseFormat;
 exports.preprocessDoubaoBboxJson = __webpack_exports__.preprocessDoubaoBboxJson;
 exports.safeParseJson = __webpack_exports__.safeParseJson;
 for(var __webpack_i__ in __webpack_exports__)if (-1 === [
-    "call",
-    "callAiFnWithStringResponse",
-    "callToGetJSONObject",
+    "callAI",
+    "callAIWithObjectResponse",
+    "callAIWithStringResponse",
     "extractJSONFromCodeBlock",
     "getResponseFormat",
     "preprocessDoubaoBboxJson",