@midscene/core 1.10.2 → 1.10.3

package/dist/es/report.mjs

@@ -3,7 +3,7 @@ import { basename, dirname, join, resolve } from "node:path";
 import { getMidsceneRunSubDir } from "@midscene/shared/common";
 import { antiEscapeScriptTag, logMsg } from "@midscene/shared/utils";
 import { getReportFileName } from "./agent/index.mjs";
-import { DATA_SCREENSHOT_MODE_ATTR, extractAllDumpScriptsSync, extractLastDumpScriptSync, getBaseUrlFixScript, streamDumpScriptsSync, streamImageScriptsToFile } from "./dump/html-utils.mjs";
+import { DATA_SCREENSHOT_MODE_ATTR, extractAllDumpScriptsSync, extractLastDumpScriptSync, generateAgentReportComment, getBaseUrlFixScript, streamDumpScriptsSync, streamImageScriptsToFile } from "./dump/html-utils.mjs";
 import { normalizeScreenshotRef, resolveScreenshotSource } from "./dump/screenshot-store.mjs";
 import { ReportActionDump } from "./types.mjs";
 import { getReportTpl, getVersion, reportHTMLContent } from "./utils.mjs";
@@ -59,6 +59,29 @@ function peekReportSdkVersion(reportFilePath) {
     }
 }
 const warnedMismatchedVersions = new Set();
+function tryParseAgentReportDump(dumpString) {
+    const trimmed = dumpString.trimStart();
+    if (!trimmed.startsWith('{') || !trimmed.includes('"executions"')) return null;
+    try {
+        return ReportActionDump.fromSerializedString(trimmed);
+    } catch  {
+        return null;
+    }
+}
+function mergedAgentReportComment(reports) {
+    if (0 === reports.length) return '';
+    if (1 === reports.length) return generateAgentReportComment(reports[0]);
+    const deviceTypes = Array.from(new Set(reports.map((report)=>report.deviceType).filter(Boolean)));
+    const mergedReport = new ReportActionDump({
+        sdkVersion: reports[0].sdkVersion || getVersion(),
+        groupName: 'Merged Midscene Report',
+        groupDescription: 'Agent-readable summary for merged report HTML',
+        modelBriefs: reports.flatMap((report)=>report.modelBriefs ?? []),
+        deviceType: 1 === deviceTypes.length ? deviceTypes[0] : 'mixed',
+        executions: reports.flatMap((report)=>report.executions ?? [])
+    });
+    return generateAgentReportComment(mergedReport);
+}
 class ReportMergingTool {
     createEmptyDumpString(groupName, groupDescription) {
         return new ReportActionDump({
@@ -131,6 +154,7 @@ class ReportMergingTool {
             const tplWithoutClose = -1 !== htmlEndIdx ? tpl.slice(0, htmlEndIdx) : tpl;
             appendFileSync(outputFilePath, tplWithoutClose);
             if (hasDirectoryModeReport) appendFileSync(outputFilePath, getBaseUrlFixScript());
+            const agentReports = [];
             for(let i = 0; i < this.reportInfos.length; i++){
                 const reportInfo = this.reportInfos[i];
                 logMsg(`Processing report ${i + 1}/${this.reportInfos.length}`);
@@ -159,6 +183,8 @@ class ReportMergingTool {
                     const extractedDumpString = allDumps.length > 0 ? this.mergeDumpScripts(allDumps.map((d)=>d.content)) : extractLastDumpScriptSync(reportInfo.reportFilePath);
                     if (extractedDumpString) dumpString = extractedDumpString;
                 }
+                const agentReport = tryParseAgentReportDump(dumpString);
+                if (agentReport) agentReports.push(agentReport);
                 const reportHtmlStr = `${reportHTMLContent({
                     dumpString,
                     attributes: {
@@ -174,6 +200,8 @@ class ReportMergingTool {
                 }, void 0, void 0, false)}\n`;
                 appendFileSync(outputFilePath, reportHtmlStr);
             }
+            const agentComment = mergedAgentReportComment(agentReports);
+            if (agentComment) appendFileSync(outputFilePath, agentComment);
             appendFileSync(outputFilePath, `${htmlEndTag}\n`);
             logMsg(`Successfully merged new report: ${outputFilePath}`);
             if (rmOriginalReports) {

package/dist/es/report.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"report.mjs","sources":["../../src/report.ts"],"sourcesContent":["import {\n  appendFileSync,\n  copyFileSync,\n  existsSync,\n  mkdirSync,\n  readdirSync,\n  rmSync,\n  unlinkSync,\n  writeFileSync,\n} from 'node:fs';\nimport * as path from 'node:path';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { antiEscapeScriptTag, logMsg } from '@midscene/shared/utils';\nimport { getReportFileName } from './agent';\nimport {\n  DATA_SCREENSHOT_MODE_ATTR,\n  extractAllDumpScriptsSync,\n  extractLastDumpScriptSync,\n  getBaseUrlFixScript,\n  streamDumpScriptsSync,\n  streamImageScriptsToFile,\n} from './dump/html-utils';\nimport {\n  normalizeScreenshotRef,\n  resolveScreenshotSource,\n} from './dump/screenshot-store';\nimport {\n  type ExecutionDump,\n  type IExecutionDump,\n  ReportActionDump,\n  type ScreenshotMode,\n} from './types';\nimport type { ReportFileWithAttributes } from './types';\nimport { getReportTpl, getVersion, reportHTMLContent } from './utils';\n\n/**\n * Read the screenshot storage mode a report declared at generation time.\n *\n * Reports written by current versions stamp `data-screenshot-mode` onto every\n * dump script tag, so we only need to peek at the first real dump script (the\n * template's bundled JS also references the dump type string, hence the\n * `data-group-id` filter) and can stop streaming immediately.\n *\n * Returns undefined for legacy reports that predate the attribute or for\n * unreadable files, letting the caller fall back to a filesystem heuristic.\n */\nconst screenshotModeAttrRegExp = new RegExp(\n  `${DATA_SCREENSHOT_MODE_ATTR}=\"(inline|directory)\"`,\n);\n\nfunction readDeclaredScreenshotMode(\n  reportFilePath: string,\n): ScreenshotMode | undefined {\n  let mode: ScreenshotMode | undefined;\n  try {\n    streamDumpScriptsSync(reportFilePath, ({ openTag }) => {\n      // Skip false matches from the template's bundled JS code.\n      if (!openTag.includes('data-group-id')) return false;\n      const match = openTag.match(screenshotModeAttrRegExp);\n      mode = match?.[1] as ScreenshotMode | undefined;\n      return true; // the first real dump script decides the mode\n    });\n  } catch {\n    // Unreadable / non-existent file — let the caller fall back.\n  }\n  return mode;\n}\n\n/**\n * Check if a report is in directory mode (html-and-external-assets).\n * Directory mode reports: {name}/index.html + {name}/screenshots/\n *\n * The mode is read from the report's own `data-screenshot-mode` metadata, which\n * is authoritative regardless of whether the run happened to capture any\n * screenshots. For legacy reports without the attribute we fall back to the old\n * filesystem heuristic (an `index.html` that has a sibling `screenshots/` dir).\n */\nexport function isDirectoryModeReport(reportFilePath: string): boolean {\n  // Directory-mode reports are always written as `{name}/index.html`, so any\n  // other filename is inline. Short-circuit before touching the file so the\n  // common inline case (`{name}.html`) costs nothing.\n  if (path.basename(reportFilePath) !== 'index.html') return false;\n\n  const declared = readDeclaredScreenshotMode(reportFilePath);\n  if (declared) return declared === 'directory';\n\n  // Legacy fallback for reports generated before screenshotMode was embedded.\n  return existsSync(path.join(path.dirname(reportFilePath), 'screenshots'));\n}\n\n/**\n * Whether a report lives in its own dedicated directory (`{name}/index.html`)\n * rather than being a single standalone file (`{name}.html`).\n *\n * This is a structural fact about *where the report file sits*, independent of\n * how it stores screenshots: a directory report can keep screenshots external\n * (a `screenshots/` sibling) OR inline them into `index.html`. Deletion needs\n * this — not the screenshot mode — to decide whether to remove the whole\n * directory or just unlink a file, so that an inline-screenshot report nested in\n * its own folder still has the folder removed instead of being orphaned.\n */\nfunction isDirectoryBasedReport(reportFilePath: string): boolean {\n  return path.basename(reportFilePath) === 'index.html';\n}\n\n/**\n * Deduplicate executions by stable id, keeping only the last occurrence.\n * Old-format executions without id are always preserved.\n */\nexport function dedupeExecutionsKeepLatest<T extends Pick<ExecutionDump, 'id'>>(\n  executions: T[],\n): T[] {\n  let noIdCounter = 0;\n  const deduped = new Map<string, T>();\n  for (const exec of executions) {\n    const key = exec.id || `__no_id_${noIdCounter++}`;\n    deduped.set(key, exec);\n  }\n  return Array.from(deduped.values());\n}\n/**\n * Peek at the first `sdkVersion` field embedded in a midscene_web_dump\n * script tag inside the given report file. Returns undefined if no\n * recognizable tag or sdkVersion is present.\n */\nfunction peekReportSdkVersion(reportFilePath: string): string | undefined {\n  try {\n    const dump = extractLastDumpScriptSync(reportFilePath);\n    if (!dump) return undefined;\n    const match = dump.match(/\"sdkVersion\"\\s*:\\s*\"([^\"]+)\"/);\n    return match?.[1];\n  } catch {\n    return undefined;\n  }\n}\n\nconst warnedMismatchedVersions = new Set<string>();\n\nexport class ReportMergingTool {\n  private reportInfos: ReportFileWithAttributes[] = [];\n\n  private createEmptyDumpString(groupName: string, groupDescription?: string) {\n    return new ReportActionDump({\n      sdkVersion: '',\n      groupName,\n      groupDescription,\n      modelBriefs: [],\n      executions: [],\n    }).serialize();\n  }\n\n  public append(reportInfo: ReportFileWithAttributes) {\n    if (reportInfo.reportFilePath) {\n      const sourceVersion = peekReportSdkVersion(reportInfo.reportFilePath);\n      const currentVersion = getVersion();\n      if (\n        sourceVersion &&\n        currentVersion &&\n        sourceVersion !== currentVersion &&\n        !warnedMismatchedVersions.has(sourceVersion)\n      ) {\n        warnedMismatchedVersions.add(sourceVersion);\n        logMsg(\n          `[@midscene/core] ReportMergingTool version mismatch: source report was written by @midscene/core@${sourceVersion} but the merger is @midscene/core@${currentVersion}. This commonly means @midscene/core and the device package (e.g. @midscene/android) resolve to different versions in node_modules. Merged output may silently drop intermediate steps. Align the versions and reinstall (rm -rf node_modules package-lock.json && npm install).`,\n        );\n      }\n    }\n    this.reportInfos.push(reportInfo);\n  }\n  public clear() {\n    this.reportInfos = [];\n  }\n\n  /**\n   * Merge multiple dump script contents (from the same source report)\n   * into a single serialized ReportActionDump string.\n   * If there's only one dump, return it as-is. If multiple, merge\n   * all executions into the first dump's group structure.\n   */\n  private mergeDumpScripts(contents: string[]): string {\n    const unescaped = contents\n      .map((c) => antiEscapeScriptTag(c))\n      .filter((c) => c.length > 0);\n    if (unescaped.length === 0) return '';\n    if (unescaped.length === 1) return unescaped[0];\n\n    // Parse all dumps and collect executions, deduplicating by id (keep last).\n    // Only executions with a stable id are deduped; old-format entries without\n    // id are always kept (they may be distinct despite sharing the same name).\n    const base = ReportActionDump.fromSerializedString(unescaped[0]);\n    const allExecutions = [...base.executions];\n    for (let i = 1; i < unescaped.length; i++) {\n      const other = ReportActionDump.fromSerializedString(unescaped[i]);\n      allExecutions.push(...other.executions);\n    }\n    base.executions = dedupeExecutionsKeepLatest(allExecutions);\n    return base.serialize();\n  }\n\n  public mergeReports(\n    reportFileName: 'AUTO' | string = 'AUTO',\n    opts?: {\n      rmOriginalReports?: boolean;\n      overwrite?: boolean;\n      outputDir?: string;\n    },\n  ): string | null {\n    const {\n      rmOriginalReports = false,\n      overwrite = false,\n      outputDir,\n    } = opts ?? {};\n\n    if (this.reportInfos.length === 0) {\n      logMsg('No reports to merge');\n      return null;\n    }\n\n    const targetDir = outputDir\n      ? path.resolve(outputDir)\n      : getMidsceneRunSubDir('report');\n    if (outputDir) {\n      mkdirSync(targetDir, { recursive: true });\n    }\n\n    // Resolve each report's screenshot mode exactly once. isDirectoryModeReport\n    // reads the file to find the authoritative metadata, so recomputing it for\n    // both the output-path decision and the per-report merge loop would re-scan\n    // every report twice.\n    const isDirModeByIndex = this.reportInfos.map((info) =>\n      Boolean(\n        info.reportFilePath && isDirectoryModeReport(info.reportFilePath),\n      ),\n    );\n    const hasDirectoryModeReport = isDirModeByIndex.some(Boolean);\n\n    const resolvedName =\n      reportFileName === 'AUTO'\n        ? getReportFileName('merged-report')\n        : reportFileName;\n\n    // Directory mode: output as {name}/index.html to keep relative paths working\n    // Inline mode: output as {name}.html (single file)\n    const outputFilePath = hasDirectoryModeReport\n      ? path.resolve(targetDir, resolvedName, 'index.html')\n      : path.resolve(targetDir, `${resolvedName}.html`);\n\n    if (reportFileName !== 'AUTO' && existsSync(outputFilePath)) {\n      if (!overwrite) {\n        throw new Error(\n          `Report file already exists: ${outputFilePath}\\nSet overwrite to true to overwrite this file.`,\n        );\n      }\n      if (hasDirectoryModeReport) {\n        rmSync(path.dirname(outputFilePath), { recursive: true, force: true });\n      } else {\n        unlinkSync(outputFilePath);\n      }\n    }\n\n    if (hasDirectoryModeReport) {\n      mkdirSync(path.dirname(outputFilePath), { recursive: true });\n    }\n\n    logMsg(\n      `Start merging ${this.reportInfos.length} reports...\\nCreating template file...`,\n    );\n\n    try {\n      // Write template without closing </html> tag so we can append\n      // dump scripts before it. The closing tag is added at the end.\n      const htmlEndTag = '</html>';\n      const tpl = getReportTpl();\n      const htmlEndIdx = tpl.lastIndexOf(htmlEndTag);\n      const tplWithoutClose =\n        htmlEndIdx !== -1 ? tpl.slice(0, htmlEndIdx) : tpl;\n      appendFileSync(outputFilePath, tplWithoutClose);\n\n      // For directory-mode output, inject base URL fix script\n      if (hasDirectoryModeReport) {\n        appendFileSync(outputFilePath, getBaseUrlFixScript());\n      }\n\n      // Process all reports one by one\n      for (let i = 0; i < this.reportInfos.length; i++) {\n        const reportInfo = this.reportInfos[i];\n        logMsg(`Processing report ${i + 1}/${this.reportInfos.length}`);\n\n        const { reportAttributes } = reportInfo;\n        let dumpString = this.createEmptyDumpString(\n          reportAttributes.testTitle,\n          reportAttributes.testDescription,\n        );\n        let mergedGroupId = `merged-group-${i}`;\n\n        if (reportInfo.reportFilePath) {\n          if (isDirModeByIndex[i]) {\n            // Directory mode: copy external screenshot files. A directory-mode\n            // report can legitimately have no screenshots/ dir (a run that\n            // captured nothing), so only copy when the source dir exists.\n            const reportDir = path.dirname(reportInfo.reportFilePath);\n            const screenshotsDir = path.join(reportDir, 'screenshots');\n            if (existsSync(screenshotsDir)) {\n              const mergedScreenshotsDir = path.join(\n                path.dirname(outputFilePath),\n                'screenshots',\n              );\n              mkdirSync(mergedScreenshotsDir, { recursive: true });\n              for (const file of readdirSync(screenshotsDir)) {\n                const src = path.join(screenshotsDir, file);\n                const dest = path.join(mergedScreenshotsDir, file);\n                copyFileSync(src, dest);\n              }\n            }\n          } else {\n            // Inline mode: stream image scripts to output file\n            streamImageScriptsToFile(reportInfo.reportFilePath, outputFilePath);\n          }\n\n          // Extract all dump scripts from the source report.\n          // After the per-execution append refactor, a single source report\n          // may contain multiple <script type=\"midscene_web_dump\"> tags\n          // (one per execution). We merge them into a single ReportActionDump.\n          // Filter by data-group-id to exclude false matches from the template's\n          // bundled JS code, which also references the midscene_web_dump type string.\n          const allDumps = extractAllDumpScriptsSync(\n            reportInfo.reportFilePath,\n          ).filter((d) => d.openTag.includes('data-group-id'));\n          const groupIdMatch = allDumps[0]?.openTag.match(\n            /data-group-id=\"([^\"]+)\"/,\n          );\n          if (groupIdMatch) {\n            mergedGroupId = decodeURIComponent(groupIdMatch[1]);\n          }\n          const extractedDumpString =\n            allDumps.length > 0\n              ? this.mergeDumpScripts(allDumps.map((d) => d.content))\n              : extractLastDumpScriptSync(reportInfo.reportFilePath);\n          if (extractedDumpString) {\n            dumpString = extractedDumpString;\n          }\n        }\n\n        const reportHtmlStr = `${reportHTMLContent(\n          {\n            dumpString,\n            attributes: {\n              'data-group-id': mergedGroupId,\n              [DATA_SCREENSHOT_MODE_ATTR]: hasDirectoryModeReport\n                ? 'directory'\n                : 'inline',\n              playwright_test_duration: reportAttributes.testDuration,\n              playwright_test_status: reportAttributes.testStatus,\n              playwright_test_title: reportAttributes.testTitle,\n              playwright_test_id: reportAttributes.testId,\n              playwright_test_description: reportAttributes.testDescription,\n              is_merged: true,\n            },\n          },\n          undefined,\n          undefined,\n          false,\n        )}\\n`;\n\n        appendFileSync(outputFilePath, reportHtmlStr);\n      }\n\n      // Close the HTML document\n      appendFileSync(outputFilePath, `${htmlEndTag}\\n`);\n\n      logMsg(`Successfully merged new report: ${outputFilePath}`);\n\n      // Remove original reports if needed\n      if (rmOriginalReports) {\n        for (const info of this.reportInfos) {\n          if (!info.reportFilePath) continue;\n          try {\n            if (isDirectoryBasedReport(info.reportFilePath)) {\n              // The report owns its directory (`{name}/index.html`) — remove the\n              // whole folder, whether screenshots are external or inlined.\n              const reportDir = path.dirname(info.reportFilePath);\n              rmSync(reportDir, { recursive: true, force: true });\n            } else {\n              unlinkSync(info.reportFilePath);\n            }\n          } catch (error) {\n            logMsg(`Error deleting report ${info.reportFilePath}: ${error}`);\n          }\n        }\n        logMsg(`Removed ${this.reportInfos.length} original reports`);\n      }\n      return outputFilePath;\n    } catch (error) {\n      logMsg(`Error in mergeReports: ${error}`);\n      throw error;\n    }\n  }\n}\n\nexport interface SplitReportHtmlOptions {\n  htmlPath: string;\n  outputDir: string;\n}\n\nexport interface SplitReportHtmlResult {\n  executionJsonFiles: string[];\n  screenshotFiles: string[];\n}\n\nexport interface CollectedReportExecutions {\n  baseDump: ReportActionDump;\n  executions: IExecutionDump[];\n}\n\n/**\n * Collect executions from a report HTML, deduplicating by stable id while\n * keeping only the latest occurrence. Old-format executions without id are\n * always preserved.\n */\nexport function collectDedupedExecutions(\n  htmlPath: string,\n): CollectedReportExecutions {\n  let baseDump: ReportActionDump | null = null;\n  let executionSerial = 0;\n  const latestSerialByExecutionId = new Map<string, number>();\n\n  streamDumpScriptsSync(htmlPath, (dumpScript) => {\n    if (!dumpScript.openTag.includes('data-group-id')) {\n      return false;\n    }\n    const groupedDump = ReportActionDump.fromSerializedString(\n      antiEscapeScriptTag(dumpScript.content),\n    );\n    for (const execution of groupedDump.executions) {\n      executionSerial += 1;\n      if (execution.id) {\n        latestSerialByExecutionId.set(execution.id, executionSerial);\n      }\n    }\n    return false;\n  });\n\n  const executions: IExecutionDump[] = [];\n  executionSerial = 0;\n  streamDumpScriptsSync(htmlPath, (dumpScript) => {\n    if (!dumpScript.openTag.includes('data-group-id')) {\n      return false;\n    }\n\n    const groupedDump = ReportActionDump.fromSerializedString(\n      antiEscapeScriptTag(dumpScript.content),\n    );\n    if (!baseDump) {\n      baseDump = groupedDump;\n    }\n\n    for (const execution of groupedDump.executions) {\n      executionSerial += 1;\n      if (\n        execution.id &&\n        latestSerialByExecutionId.get(execution.id) !== executionSerial\n      ) {\n        continue;\n      }\n      executions.push(execution);\n    }\n\n    return false;\n  });\n\n  if (!baseDump) {\n    throw new Error(`No report dump scripts found in ${htmlPath}`);\n  }\n\n  return {\n    baseDump,\n    executions,\n  };\n}\n\nfunction extensionByMimeType(mimeType: string): 'png' | 'jpeg' {\n  if (mimeType === 'image/png') return 'png';\n  if (mimeType === 'image/jpeg') return 'jpeg';\n  throw new Error(`Unsupported screenshot mime type: ${mimeType}`);\n}\n\nfunction externalizeScreenshotsInExecution(\n  execution: IExecutionDump,\n  opts: {\n    htmlPath: string;\n    screenshotsDir: string;\n    writtenFiles: Set<string>;\n  },\n): void {\n  const visit = (node: unknown): void => {\n    if (Array.isArray(node)) {\n      for (const item of node) {\n        visit(item);\n      }\n      return;\n    }\n\n    if (typeof node !== 'object' || node === null) return;\n\n    const ref = normalizeScreenshotRef(node);\n    if (ref) {\n      const ext = extensionByMimeType(ref.mimeType);\n      const fileName = `${ref.id}.${ext}`;\n      const relativePath = `./screenshots/${fileName}`;\n      const absolutePath = path.join(opts.screenshotsDir, fileName);\n\n      if (!opts.writtenFiles.has(fileName)) {\n        const resolved = resolveScreenshotSource(ref, {\n          reportPath: opts.htmlPath,\n        });\n        if (resolved.type === 'data-uri') {\n          const rawBase64 = resolved.dataUri.replace(\n            /^data:image\\/[a-zA-Z+]+;base64,/,\n            '',\n          );\n          writeFileSync(absolutePath, Buffer.from(rawBase64, 'base64'));\n        } else {\n          copyFileSync(resolved.filePath, absolutePath);\n        }\n        opts.writtenFiles.add(fileName);\n      }\n\n      ref.storage = 'file';\n      ref.path = relativePath;\n      return;\n    }\n\n    for (const value of Object.values(node)) {\n      visit(value);\n    }\n  };\n\n  visit(execution);\n}\n\n/**\n * Reverse parse a Midscene report HTML into per-execution JSON files and\n * externalized screenshots.\n */\nexport function splitReportHtmlByExecution(\n  options: SplitReportHtmlOptions,\n): SplitReportHtmlResult {\n  const { htmlPath, outputDir } = options;\n  const screenshotsDir = path.join(outputDir, 'screenshots');\n\n  mkdirSync(outputDir, { recursive: true });\n  mkdirSync(screenshotsDir, { recursive: true });\n\n  const executionJsonFiles: string[] = [];\n  const writtenScreenshotFiles = new Set<string>();\n  const { baseDump, executions } = collectDedupedExecutions(htmlPath);\n\n  let fileIndex = 0;\n  for (const execution of executions) {\n    fileIndex += 1;\n    externalizeScreenshotsInExecution(execution, {\n      htmlPath,\n      screenshotsDir,\n      writtenFiles: writtenScreenshotFiles,\n    });\n    const singleExecutionDump = new ReportActionDump({\n      sdkVersion: baseDump.sdkVersion,\n      groupName: baseDump.groupName,\n      groupDescription: baseDump.groupDescription,\n      modelBriefs: baseDump.modelBriefs,\n      deviceType: baseDump.deviceType,\n      executions: [execution],\n    });\n\n    const jsonFilePath = path.join(outputDir, `${fileIndex}.execution.json`);\n    writeFileSync(jsonFilePath, singleExecutionDump.serialize(2), 'utf-8');\n    executionJsonFiles.push(jsonFilePath);\n  }\n\n  return {\n    executionJsonFiles,\n    screenshotFiles: Array.from(writtenScreenshotFiles)\n      .sort()\n      .map((fileName) => path.join(screenshotsDir, fileName)),\n  };\n}\n"],"names":["screenshotModeAttrRegExp","RegExp","DATA_SCREENSHOT_MODE_ATTR","readDeclaredScreenshotMode","reportFilePath","mode","streamDumpScriptsSync","openTag","match","isDirectoryModeReport","path","declared","existsSync","isDirectoryBasedReport","dedupeExecutionsKeepLatest","executions","noIdCounter","deduped","Map","exec","key","Array","peekReportSdkVersion","dump","extractLastDumpScriptSync","warnedMismatchedVersions","Set","ReportMergingTool","groupName","groupDescription","ReportActionDump","reportInfo","sourceVersion","currentVersion","getVersion","logMsg","contents","unescaped","c","antiEscapeScriptTag","base","allExecutions","i","other","reportFileName","opts","rmOriginalReports","overwrite","outputDir","targetDir","getMidsceneRunSubDir","mkdirSync","isDirModeByIndex","info","Boolean","hasDirectoryModeReport","resolvedName","getReportFileName","outputFilePath","Error","rmSync","unlinkSync","htmlEndTag","tpl","getReportTpl","htmlEndIdx","tplWithoutClose","appendFileSync","getBaseUrlFixScript","reportAttributes","dumpString","mergedGroupId","reportDir","screenshotsDir","mergedScreenshotsDir","file","readdirSync","src","dest","copyFileSync","streamImageScriptsToFile","allDumps","extractAllDumpScriptsSync","d","groupIdMatch","decodeURIComponent","extractedDumpString","reportHtmlStr","reportHTMLContent","undefined","error","collectDedupedExecutions","htmlPath","baseDump","executionSerial","latestSerialByExecutionId","dumpScript","groupedDump","execution","extensionByMimeType","mimeType","externalizeScreenshotsInExecution","visit","node","item","ref","normalizeScreenshotRef","ext","fileName","relativePath","absolutePath","resolved","resolveScreenshotSource","rawBase64","writeFileSync","Buffer","value","Object","splitReportHtmlByExecution","options","executionJsonFiles","writtenScreenshotFiles","fileIndex","singleExecutionDump","jsonFilePath"],"mappings":";;;;;;;;;;;;;;;;;;;AA8CA,MAAMA,2BAA2B,IAAIC,OACnC,GAAGC,0BAA0B,qBAAqB,CAAC;AAGrD,SAASC,2BACPC,cAAsB;IAEtB,IAAIC;IACJ,IAAI;QACFC,sBAAsBF,gBAAgB,CAAC,EAAEG,OAAO,EAAE;YAEhD,IAAI,CAACA,QAAQ,QAAQ,CAAC,kBAAkB,OAAO;YAC/C,MAAMC,QAAQD,QAAQ,KAAK,CAACP;YAC5BK,OAAOG,OAAO,CAAC,EAAE;YACjB,OAAO;QACT;IACF,EAAE,OAAM,CAER;IACA,OAAOH;AACT;AAWO,SAASI,sBAAsBL,cAAsB;IAI1D,IAAIM,AAAkC,iBAAlCA,SAAcN,iBAAkC,OAAO;IAE3D,MAAMO,WAAWR,2BAA2BC;IAC5C,IAAIO,UAAU,OAAOA,AAAa,gBAAbA;IAGrB,OAAOC,WAAWF,KAAUA,QAAaN,iBAAiB;AAC5D;AAaA,SAASS,uBAAuBT,cAAsB;IACpD,OAAOM,AAAkC,iBAAlCA,SAAcN;AACvB;AAMO,SAASU,2BACdC,UAAe;IAEf,IAAIC,cAAc;IAClB,MAAMC,UAAU,IAAIC;IACpB,KAAK,MAAMC,QAAQJ,WAAY;QAC7B,MAAMK,MAAMD,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAEH,eAAe;QACjDC,QAAQ,GAAG,CAACG,KAAKD;IACnB;IACA,OAAOE,MAAM,IAAI,CAACJ,QAAQ,MAAM;AAClC;AAMA,SAASK,qBAAqBlB,cAAsB;IAClD,IAAI;QACF,MAAMmB,OAAOC,0BAA0BpB;QACvC,IAAI,CAACmB,MAAM;QACX,MAAMf,QAAQe,KAAK,KAAK,CAAC;QACzB,OAAOf,OAAO,CAAC,EAAE;IACnB,EAAE,OAAM;QACN;IACF;AACF;AAEA,MAAMiB,2BAA2B,IAAIC;AAE9B,MAAMC;IAGH,sBAAsBC,SAAiB,EAAEC,gBAAyB,EAAE;QAC1E,OAAO,IAAIC,iBAAiB;YAC1B,YAAY;YACZF;YACAC;YACA,aAAa,EAAE;YACf,YAAY,EAAE;QAChB,GAAG,SAAS;IACd;IAEO,OAAOE,UAAoC,EAAE;QAClD,IAAIA,WAAW,cAAc,EAAE;YAC7B,MAAMC,gBAAgBV,qBAAqBS,WAAW,cAAc;YACpE,MAAME,iBAAiBC;YACvB,IACEF,iBACAC,kBACAD,kBAAkBC,kBAClB,CAACR,yBAAyB,GAAG,CAACO,gBAC9B;gBACAP,yBAAyB,GAAG,CAACO;gBAC7BG,OACE,CAAC,iGAAiG,EAAEH,cAAc,kCAAkC,EAAEC,eAAe,gRAAgR,CAAC;YAE1b;QACF;QACA,IAAI,CAAC,WAAW,CAAC,IAAI,CAACF;IACxB;IACO,QAAQ;QACb,IAAI,CAAC,WAAW,GAAG,EAAE;IACvB;IAQQ,iBAAiBK,QAAkB,EAAU;QACnD,MAAMC,YAAYD,SACf,GAAG,CAAC,CAACE,IAAMC,oBAAoBD,IAC/B,MAAM,CAAC,CAACA,IAAMA,EAAE,MAAM,GAAG;QAC5B,IAAID,AAAqB,MAArBA,UAAU,MAAM,EAAQ,OAAO;QACnC,IAAIA,AAAqB,MAArBA,UAAU,MAAM,EAAQ,OAAOA,SAAS,CAAC,EAAE;QAK/C,MAAMG,OAAOV,iBAAiB,oBAAoB,CAACO,SAAS,CAAC,EAAE;QAC/D,MAAMI,gBAAgB;eAAID,KAAK,UAAU;SAAC;QAC1C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,UAAU,MAAM,EAAEK,IAAK;YACzC,MAAMC,QAAQb,iBAAiB,oBAAoB,CAACO,SAAS,CAACK,EAAE;YAChED,cAAc,IAAI,IAAIE,MAAM,UAAU;QACxC;QACAH,KAAK,UAAU,GAAG1B,2BAA2B2B;QAC7C,OAAOD,KAAK,SAAS;IACvB;IAEO,aACLI,iBAAkC,MAAM,EACxCC,IAIC,EACc;QACf,MAAM,EACJC,oBAAoB,KAAK,EACzBC,YAAY,KAAK,EACjBC,SAAS,EACV,GAAGH,QAAQ,CAAC;QAEb,IAAI,AAA4B,MAA5B,IAAI,CAAC,WAAW,CAAC,MAAM,EAAQ;YACjCV,OAAO;YACP,OAAO;QACT;QAEA,MAAMc,YAAYD,YACdtC,QAAasC,aACbE,qBAAqB;QACzB,IAAIF,WACFG,UAAUF,WAAW;YAAE,WAAW;QAAK;QAOzC,MAAMG,mBAAmB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAACC,OAC7CC,QACED,KAAK,cAAc,IAAI5C,sBAAsB4C,KAAK,cAAc;QAGpE,MAAME,yBAAyBH,iBAAiB,IAAI,CAACE;QAErD,MAAME,eACJZ,AAAmB,WAAnBA,iBACIa,kBAAkB,mBAClBb;QAIN,MAAMc,iBAAiBH,yBACnB7C,QAAauC,WAAWO,cAAc,gBACtC9C,QAAauC,WAAW,GAAGO,aAAa,KAAK,CAAC;QAElD,IAAIZ,AAAmB,WAAnBA,kBAA6BhC,WAAW8C,iBAAiB;YAC3D,IAAI,CAACX,WACH,MAAM,IAAIY,MACR,CAAC,4BAA4B,EAAED,eAAe,+CAA+C,CAAC;YAGlG,IAAIH,wBACFK,OAAOlD,QAAagD,iBAAiB;gBAAE,WAAW;gBAAM,OAAO;YAAK;iBAEpEG,WAAWH;QAEf;QAEA,IAAIH,wBACFJ,UAAUzC,QAAagD,iBAAiB;YAAE,WAAW;QAAK;QAG5DvB,OACE,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,sCAAsC,CAAC;QAGlF,IAAI;YAGF,MAAM2B,aAAa;YACnB,MAAMC,MAAMC;YACZ,MAAMC,aAAaF,IAAI,WAAW,CAACD;YACnC,MAAMI,kBACJD,AAAe,OAAfA,aAAoBF,IAAI,KAAK,CAAC,GAAGE,cAAcF;YACjDI,eAAeT,gBAAgBQ;YAG/B,IAAIX,wBACFY,eAAeT,gBAAgBU;YAIjC,IAAK,IAAI1B,IAAI,GAAGA,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAEA,IAAK;gBAChD,MAAMX,aAAa,IAAI,CAAC,WAAW,CAACW,EAAE;gBACtCP,OAAO,CAAC,kBAAkB,EAAEO,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;gBAE9D,MAAM,EAAE2B,gBAAgB,EAAE,GAAGtC;gBAC7B,IAAIuC,aAAa,IAAI,CAAC,qBAAqB,CACzCD,iBAAiB,SAAS,EAC1BA,iBAAiB,eAAe;gBAElC,IAAIE,gBAAgB,CAAC,aAAa,EAAE7B,GAAG;gBAEvC,IAAIX,WAAW,cAAc,EAAE;oBAC7B,IAAIqB,gBAAgB,CAACV,EAAE,EAAE;wBAIvB,MAAM8B,YAAY9D,QAAaqB,WAAW,cAAc;wBACxD,MAAM0C,iBAAiB/D,KAAU8D,WAAW;wBAC5C,IAAI5D,WAAW6D,iBAAiB;4BAC9B,MAAMC,uBAAuBhE,KAC3BA,QAAagD,iBACb;4BAEFP,UAAUuB,sBAAsB;gCAAE,WAAW;4BAAK;4BAClD,KAAK,MAAMC,QAAQC,YAAYH,gBAAiB;gCAC9C,MAAMI,MAAMnE,KAAU+D,gBAAgBE;gCACtC,MAAMG,OAAOpE,KAAUgE,sBAAsBC;gCAC7CI,aAAaF,KAAKC;4BACpB;wBACF;oBACF,OAEEE,yBAAyBjD,WAAW,cAAc,EAAE2B;oBAStD,MAAMuB,WAAWC,0BACfnD,WAAW,cAAc,EACzB,MAAM,CAAC,CAACoD,IAAMA,EAAE,OAAO,CAAC,QAAQ,CAAC;oBACnC,MAAMC,eAAeH,QAAQ,CAAC,EAAE,EAAE,QAAQ,MACxC;oBAEF,IAAIG,cACFb,gBAAgBc,mBAAmBD,YAAY,CAAC,EAAE;oBAEpD,MAAME,sBACJL,SAAS,MAAM,GAAG,IACd,IAAI,CAAC,gBAAgB,CAACA,SAAS,GAAG,CAAC,CAACE,IAAMA,EAAE,OAAO,KACnD3D,0BAA0BO,WAAW,cAAc;oBACzD,IAAIuD,qBACFhB,aAAagB;gBAEjB;gBAEA,MAAMC,gBAAgB,GAAGC,kBACvB;oBACElB;oBACA,YAAY;wBACV,iBAAiBC;wBACjB,CAACrE,0BAA0B,EAAEqD,yBACzB,cACA;wBACJ,0BAA0Bc,iBAAiB,YAAY;wBACvD,wBAAwBA,iBAAiB,UAAU;wBACnD,uBAAuBA,iBAAiB,SAAS;wBACjD,oBAAoBA,iBAAiB,MAAM;wBAC3C,6BAA6BA,iBAAiB,eAAe;wBAC7D,WAAW;oBACb;gBACF,GACAoB,QACAA,QACA,OACA,EAAE,CAAC;gBAELtB,eAAeT,gBAAgB6B;YACjC;YAGApB,eAAeT,gBAAgB,GAAGI,WAAW,EAAE,CAAC;YAEhD3B,OAAO,CAAC,gCAAgC,EAAEuB,gBAAgB;YAG1D,IAAIZ,mBAAmB;gBACrB,KAAK,MAAMO,QAAQ,IAAI,CAAC,WAAW,CACjC,IAAKA,KAAK,cAAc,EACxB,IAAI;oBACF,IAAIxC,uBAAuBwC,KAAK,cAAc,GAAG;wBAG/C,MAAMmB,YAAY9D,QAAa2C,KAAK,cAAc;wBAClDO,OAAOY,WAAW;4BAAE,WAAW;4BAAM,OAAO;wBAAK;oBACnD,OACEX,WAAWR,KAAK,cAAc;gBAElC,EAAE,OAAOqC,OAAO;oBACdvD,OAAO,CAAC,sBAAsB,EAAEkB,KAAK,cAAc,CAAC,EAAE,EAAEqC,OAAO;gBACjE;gBAEFvD,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,iBAAiB,CAAC;YAC9D;YACA,OAAOuB;QACT,EAAE,OAAOgC,OAAO;YACdvD,OAAO,CAAC,uBAAuB,EAAEuD,OAAO;YACxC,MAAMA;QACR;IACF;;QAjQA,uBAAQ,eAA0C,EAAE;;AAkQtD;AAsBO,SAASC,yBACdC,QAAgB;IAEhB,IAAIC,WAAoC;IACxC,IAAIC,kBAAkB;IACtB,MAAMC,4BAA4B,IAAI7E;IAEtCZ,sBAAsBsF,UAAU,CAACI;QAC/B,IAAI,CAACA,WAAW,OAAO,CAAC,QAAQ,CAAC,kBAC/B,OAAO;QAET,MAAMC,cAAcnE,iBAAiB,oBAAoB,CACvDS,oBAAoByD,WAAW,OAAO;QAExC,KAAK,MAAME,aAAaD,YAAY,UAAU,CAAE;YAC9CH,mBAAmB;YACnB,IAAII,UAAU,EAAE,EACdH,0BAA0B,GAAG,CAACG,UAAU,EAAE,EAAEJ;QAEhD;QACA,OAAO;IACT;IAEA,MAAM/E,aAA+B,EAAE;IACvC+E,kBAAkB;IAClBxF,sBAAsBsF,UAAU,CAACI;QAC/B,IAAI,CAACA,WAAW,OAAO,CAAC,QAAQ,CAAC,kBAC/B,OAAO;QAGT,MAAMC,cAAcnE,iBAAiB,oBAAoB,CACvDS,oBAAoByD,WAAW,OAAO;QAExC,IAAI,CAACH,UACHA,WAAWI;QAGb,KAAK,MAAMC,aAAaD,YAAY,UAAU,CAAE;YAC9CH,mBAAmB;YACnB,IACEI,CAAAA,UAAU,EAAE,IACZH,0BAA0B,GAAG,CAACG,UAAU,EAAE,MAAMJ,iBAIlD/E,WAAW,IAAI,CAACmF;QAClB;QAEA,OAAO;IACT;IAEA,IAAI,CAACL,UACH,MAAM,IAAIlC,MAAM,CAAC,gCAAgC,EAAEiC,UAAU;IAG/D,OAAO;QACLC;QACA9E;IACF;AACF;AAEA,SAASoF,oBAAoBC,QAAgB;IAC3C,IAAIA,AAAa,gBAAbA,UAA0B,OAAO;IACrC,IAAIA,AAAa,iBAAbA,UAA2B,OAAO;IACtC,MAAM,IAAIzC,MAAM,CAAC,kCAAkC,EAAEyC,UAAU;AACjE;AAEA,SAASC,kCACPH,SAAyB,EACzBrD,IAIC;IAED,MAAMyD,QAAQ,CAACC;QACb,IAAIlF,MAAM,OAAO,CAACkF,OAAO;YACvB,KAAK,MAAMC,QAAQD,KACjBD,MAAME;YAER;QACF;QAEA,IAAI,AAAgB,YAAhB,OAAOD,QAAqBA,AAAS,SAATA,MAAe;QAE/C,MAAME,MAAMC,uBAAuBH;QACnC,IAAIE,KAAK;YACP,MAAME,MAAMR,oBAAoBM,IAAI,QAAQ;YAC5C,MAAMG,WAAW,GAAGH,IAAI,EAAE,CAAC,CAAC,EAAEE,KAAK;YACnC,MAAME,eAAe,CAAC,cAAc,EAAED,UAAU;YAChD,MAAME,eAAepG,KAAUmC,KAAK,cAAc,EAAE+D;YAEpD,IAAI,CAAC/D,KAAK,YAAY,CAAC,GAAG,CAAC+D,WAAW;gBACpC,MAAMG,WAAWC,wBAAwBP,KAAK;oBAC5C,YAAY5D,KAAK,QAAQ;gBAC3B;gBACA,IAAIkE,AAAkB,eAAlBA,SAAS,IAAI,EAAiB;oBAChC,MAAME,YAAYF,SAAS,OAAO,CAAC,OAAO,CACxC,mCACA;oBAEFG,cAAcJ,cAAcK,OAAO,IAAI,CAACF,WAAW;gBACrD,OACElC,aAAagC,SAAS,QAAQ,EAAED;gBAElCjE,KAAK,YAAY,CAAC,GAAG,CAAC+D;YACxB;YAEAH,IAAI,OAAO,GAAG;YACdA,IAAI,IAAI,GAAGI;YACX;QACF;QAEA,KAAK,MAAMO,SAASC,OAAO,MAAM,CAACd,MAChCD,MAAMc;IAEV;IAEAd,MAAMJ;AACR;AAMO,SAASoB,2BACdC,OAA+B;IAE/B,MAAM,EAAE3B,QAAQ,EAAE5C,SAAS,EAAE,GAAGuE;IAChC,MAAM9C,iBAAiB/D,KAAUsC,WAAW;IAE5CG,UAAUH,WAAW;QAAE,WAAW;IAAK;IACvCG,UAAUsB,gBAAgB;QAAE,WAAW;IAAK;IAE5C,MAAM+C,qBAA+B,EAAE;IACvC,MAAMC,yBAAyB,IAAI/F;IACnC,MAAM,EAAEmE,QAAQ,EAAE9E,UAAU,EAAE,GAAG4E,yBAAyBC;IAE1D,IAAI8B,YAAY;IAChB,KAAK,MAAMxB,aAAanF,WAAY;QAClC2G,aAAa;QACbrB,kCAAkCH,WAAW;YAC3CN;YACAnB;YACA,cAAcgD;QAChB;QACA,MAAME,sBAAsB,IAAI7F,iBAAiB;YAC/C,YAAY+D,SAAS,UAAU;YAC/B,WAAWA,SAAS,SAAS;YAC7B,kBAAkBA,SAAS,gBAAgB;YAC3C,aAAaA,SAAS,WAAW;YACjC,YAAYA,SAAS,UAAU;YAC/B,YAAY;gBAACK;aAAU;QACzB;QAEA,MAAM0B,eAAelH,KAAUsC,WAAW,GAAG0E,UAAU,eAAe,CAAC;QACvER,cAAcU,cAAcD,oBAAoB,SAAS,CAAC,IAAI;QAC9DH,mBAAmB,IAAI,CAACI;IAC1B;IAEA,OAAO;QACLJ;QACA,iBAAiBnG,MAAM,IAAI,CAACoG,wBACzB,IAAI,GACJ,GAAG,CAAC,CAACb,WAAalG,KAAU+D,gBAAgBmC;IACjD;AACF"}
+{"version":3,"file":"report.mjs","sources":["../../src/report.ts"],"sourcesContent":["import {\n  appendFileSync,\n  copyFileSync,\n  existsSync,\n  mkdirSync,\n  readdirSync,\n  rmSync,\n  unlinkSync,\n  writeFileSync,\n} from 'node:fs';\nimport * as path from 'node:path';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { antiEscapeScriptTag, logMsg } from '@midscene/shared/utils';\nimport { getReportFileName } from './agent';\nimport {\n  DATA_SCREENSHOT_MODE_ATTR,\n  extractAllDumpScriptsSync,\n  extractLastDumpScriptSync,\n  generateAgentReportComment,\n  getBaseUrlFixScript,\n  streamDumpScriptsSync,\n  streamImageScriptsToFile,\n} from './dump/html-utils';\nimport {\n  normalizeScreenshotRef,\n  resolveScreenshotSource,\n} from './dump/screenshot-store';\nimport {\n  type ExecutionDump,\n  type IExecutionDump,\n  ReportActionDump,\n  type ScreenshotMode,\n} from './types';\nimport type { ReportFileWithAttributes } from './types';\nimport { getReportTpl, getVersion, reportHTMLContent } from './utils';\n\n/**\n * Read the screenshot storage mode a report declared at generation time.\n *\n * Reports written by current versions stamp `data-screenshot-mode` onto every\n * dump script tag, so we only need to peek at the first real dump script (the\n * template's bundled JS also references the dump type string, hence the\n * `data-group-id` filter) and can stop streaming immediately.\n *\n * Returns undefined for legacy reports that predate the attribute or for\n * unreadable files, letting the caller fall back to a filesystem heuristic.\n */\nconst screenshotModeAttrRegExp = new RegExp(\n  `${DATA_SCREENSHOT_MODE_ATTR}=\"(inline|directory)\"`,\n);\n\nfunction readDeclaredScreenshotMode(\n  reportFilePath: string,\n): ScreenshotMode | undefined {\n  let mode: ScreenshotMode | undefined;\n  try {\n    streamDumpScriptsSync(reportFilePath, ({ openTag }) => {\n      // Skip false matches from the template's bundled JS code.\n      if (!openTag.includes('data-group-id')) return false;\n      const match = openTag.match(screenshotModeAttrRegExp);\n      mode = match?.[1] as ScreenshotMode | undefined;\n      return true; // the first real dump script decides the mode\n    });\n  } catch {\n    // Unreadable / non-existent file — let the caller fall back.\n  }\n  return mode;\n}\n\n/**\n * Check if a report is in directory mode (html-and-external-assets).\n * Directory mode reports: {name}/index.html + {name}/screenshots/\n *\n * The mode is read from the report's own `data-screenshot-mode` metadata, which\n * is authoritative regardless of whether the run happened to capture any\n * screenshots. For legacy reports without the attribute we fall back to the old\n * filesystem heuristic (an `index.html` that has a sibling `screenshots/` dir).\n */\nexport function isDirectoryModeReport(reportFilePath: string): boolean {\n  // Directory-mode reports are always written as `{name}/index.html`, so any\n  // other filename is inline. Short-circuit before touching the file so the\n  // common inline case (`{name}.html`) costs nothing.\n  if (path.basename(reportFilePath) !== 'index.html') return false;\n\n  const declared = readDeclaredScreenshotMode(reportFilePath);\n  if (declared) return declared === 'directory';\n\n  // Legacy fallback for reports generated before screenshotMode was embedded.\n  return existsSync(path.join(path.dirname(reportFilePath), 'screenshots'));\n}\n\n/**\n * Whether a report lives in its own dedicated directory (`{name}/index.html`)\n * rather than being a single standalone file (`{name}.html`).\n *\n * This is a structural fact about *where the report file sits*, independent of\n * how it stores screenshots: a directory report can keep screenshots external\n * (a `screenshots/` sibling) OR inline them into `index.html`. Deletion needs\n * this — not the screenshot mode — to decide whether to remove the whole\n * directory or just unlink a file, so that an inline-screenshot report nested in\n * its own folder still has the folder removed instead of being orphaned.\n */\nfunction isDirectoryBasedReport(reportFilePath: string): boolean {\n  return path.basename(reportFilePath) === 'index.html';\n}\n\n/**\n * Deduplicate executions by stable id, keeping only the last occurrence.\n * Old-format executions without id are always preserved.\n */\nexport function dedupeExecutionsKeepLatest<T extends Pick<ExecutionDump, 'id'>>(\n  executions: T[],\n): T[] {\n  let noIdCounter = 0;\n  const deduped = new Map<string, T>();\n  for (const exec of executions) {\n    const key = exec.id || `__no_id_${noIdCounter++}`;\n    deduped.set(key, exec);\n  }\n  return Array.from(deduped.values());\n}\n/**\n * Peek at the first `sdkVersion` field embedded in a midscene_web_dump\n * script tag inside the given report file. Returns undefined if no\n * recognizable tag or sdkVersion is present.\n */\nfunction peekReportSdkVersion(reportFilePath: string): string | undefined {\n  try {\n    const dump = extractLastDumpScriptSync(reportFilePath);\n    if (!dump) return undefined;\n    const match = dump.match(/\"sdkVersion\"\\s*:\\s*\"([^\"]+)\"/);\n    return match?.[1];\n  } catch {\n    return undefined;\n  }\n}\n\nconst warnedMismatchedVersions = new Set<string>();\n\nfunction tryParseAgentReportDump(dumpString: string): ReportActionDump | null {\n  const trimmed = dumpString.trimStart();\n  if (!trimmed.startsWith('{') || !trimmed.includes('\"executions\"')) {\n    return null;\n  }\n\n  try {\n    return ReportActionDump.fromSerializedString(trimmed);\n  } catch {\n    return null;\n  }\n}\n\nfunction mergedAgentReportComment(reports: ReportActionDump[]): string {\n  if (reports.length === 0) {\n    return '';\n  }\n  if (reports.length === 1) {\n    return generateAgentReportComment(reports[0]);\n  }\n\n  const deviceTypes = Array.from(\n    new Set(reports.map((report) => report.deviceType).filter(Boolean)),\n  );\n  const mergedReport = new ReportActionDump({\n    sdkVersion: reports[0].sdkVersion || getVersion(),\n    groupName: 'Merged Midscene Report',\n    groupDescription: 'Agent-readable summary for merged report HTML',\n    modelBriefs: reports.flatMap((report) => report.modelBriefs ?? []),\n    deviceType: deviceTypes.length === 1 ? deviceTypes[0] : 'mixed',\n    executions: reports.flatMap((report) => report.executions ?? []),\n  });\n  return generateAgentReportComment(mergedReport);\n}\n\nexport class ReportMergingTool {\n  private reportInfos: ReportFileWithAttributes[] = [];\n\n  private createEmptyDumpString(groupName: string, groupDescription?: string) {\n    return new ReportActionDump({\n      sdkVersion: '',\n      groupName,\n      groupDescription,\n      modelBriefs: [],\n      executions: [],\n    }).serialize();\n  }\n\n  public append(reportInfo: ReportFileWithAttributes) {\n    if (reportInfo.reportFilePath) {\n      const sourceVersion = peekReportSdkVersion(reportInfo.reportFilePath);\n      const currentVersion = getVersion();\n      if (\n        sourceVersion &&\n        currentVersion &&\n        sourceVersion !== currentVersion &&\n        !warnedMismatchedVersions.has(sourceVersion)\n      ) {\n        warnedMismatchedVersions.add(sourceVersion);\n        logMsg(\n          `[@midscene/core] ReportMergingTool version mismatch: source report was written by @midscene/core@${sourceVersion} but the merger is @midscene/core@${currentVersion}. This commonly means @midscene/core and the device package (e.g. @midscene/android) resolve to different versions in node_modules. Merged output may silently drop intermediate steps. Align the versions and reinstall (rm -rf node_modules package-lock.json && npm install).`,\n        );\n      }\n    }\n    this.reportInfos.push(reportInfo);\n  }\n  public clear() {\n    this.reportInfos = [];\n  }\n\n  /**\n   * Merge multiple dump script contents (from the same source report)\n   * into a single serialized ReportActionDump string.\n   * If there's only one dump, return it as-is. If multiple, merge\n   * all executions into the first dump's group structure.\n   */\n  private mergeDumpScripts(contents: string[]): string {\n    const unescaped = contents\n      .map((c) => antiEscapeScriptTag(c))\n      .filter((c) => c.length > 0);\n    if (unescaped.length === 0) return '';\n    if (unescaped.length === 1) return unescaped[0];\n\n    // Parse all dumps and collect executions, deduplicating by id (keep last).\n    // Only executions with a stable id are deduped; old-format entries without\n    // id are always kept (they may be distinct despite sharing the same name).\n    const base = ReportActionDump.fromSerializedString(unescaped[0]);\n    const allExecutions = [...base.executions];\n    for (let i = 1; i < unescaped.length; i++) {\n      const other = ReportActionDump.fromSerializedString(unescaped[i]);\n      allExecutions.push(...other.executions);\n    }\n    base.executions = dedupeExecutionsKeepLatest(allExecutions);\n    return base.serialize();\n  }\n\n  public mergeReports(\n    reportFileName: 'AUTO' | string = 'AUTO',\n    opts?: {\n      rmOriginalReports?: boolean;\n      overwrite?: boolean;\n      outputDir?: string;\n    },\n  ): string | null {\n    const {\n      rmOriginalReports = false,\n      overwrite = false,\n      outputDir,\n    } = opts ?? {};\n\n    if (this.reportInfos.length === 0) {\n      logMsg('No reports to merge');\n      return null;\n    }\n\n    const targetDir = outputDir\n      ? path.resolve(outputDir)\n      : getMidsceneRunSubDir('report');\n    if (outputDir) {\n      mkdirSync(targetDir, { recursive: true });\n    }\n\n    // Resolve each report's screenshot mode exactly once. isDirectoryModeReport\n    // reads the file to find the authoritative metadata, so recomputing it for\n    // both the output-path decision and the per-report merge loop would re-scan\n    // every report twice.\n    const isDirModeByIndex = this.reportInfos.map((info) =>\n      Boolean(\n        info.reportFilePath && isDirectoryModeReport(info.reportFilePath),\n      ),\n    );\n    const hasDirectoryModeReport = isDirModeByIndex.some(Boolean);\n\n    const resolvedName =\n      reportFileName === 'AUTO'\n        ? getReportFileName('merged-report')\n        : reportFileName;\n\n    // Directory mode: output as {name}/index.html to keep relative paths working\n    // Inline mode: output as {name}.html (single file)\n    const outputFilePath = hasDirectoryModeReport\n      ? path.resolve(targetDir, resolvedName, 'index.html')\n      : path.resolve(targetDir, `${resolvedName}.html`);\n\n    if (reportFileName !== 'AUTO' && existsSync(outputFilePath)) {\n      if (!overwrite) {\n        throw new Error(\n          `Report file already exists: ${outputFilePath}\\nSet overwrite to true to overwrite this file.`,\n        );\n      }\n      if (hasDirectoryModeReport) {\n        rmSync(path.dirname(outputFilePath), { recursive: true, force: true });\n      } else {\n        unlinkSync(outputFilePath);\n      }\n    }\n\n    if (hasDirectoryModeReport) {\n      mkdirSync(path.dirname(outputFilePath), { recursive: true });\n    }\n\n    logMsg(\n      `Start merging ${this.reportInfos.length} reports...\\nCreating template file...`,\n    );\n\n    try {\n      // Write template without closing </html> tag so we can append\n      // dump scripts before it. The closing tag is added at the end.\n      const htmlEndTag = '</html>';\n      const tpl = getReportTpl();\n      const htmlEndIdx = tpl.lastIndexOf(htmlEndTag);\n      const tplWithoutClose =\n        htmlEndIdx !== -1 ? tpl.slice(0, htmlEndIdx) : tpl;\n      appendFileSync(outputFilePath, tplWithoutClose);\n\n      // For directory-mode output, inject base URL fix script\n      if (hasDirectoryModeReport) {\n        appendFileSync(outputFilePath, getBaseUrlFixScript());\n      }\n\n      const agentReports: ReportActionDump[] = [];\n\n      // Process all reports one by one\n      for (let i = 0; i < this.reportInfos.length; i++) {\n        const reportInfo = this.reportInfos[i];\n        logMsg(`Processing report ${i + 1}/${this.reportInfos.length}`);\n\n        const { reportAttributes } = reportInfo;\n        let dumpString = this.createEmptyDumpString(\n          reportAttributes.testTitle,\n          reportAttributes.testDescription,\n        );\n        let mergedGroupId = `merged-group-${i}`;\n\n        if (reportInfo.reportFilePath) {\n          if (isDirModeByIndex[i]) {\n            // Directory mode: copy external screenshot files. A directory-mode\n            // report can legitimately have no screenshots/ dir (a run that\n            // captured nothing), so only copy when the source dir exists.\n            const reportDir = path.dirname(reportInfo.reportFilePath);\n            const screenshotsDir = path.join(reportDir, 'screenshots');\n            if (existsSync(screenshotsDir)) {\n              const mergedScreenshotsDir = path.join(\n                path.dirname(outputFilePath),\n                'screenshots',\n              );\n              mkdirSync(mergedScreenshotsDir, { recursive: true });\n              for (const file of readdirSync(screenshotsDir)) {\n                const src = path.join(screenshotsDir, file);\n                const dest = path.join(mergedScreenshotsDir, file);\n                copyFileSync(src, dest);\n              }\n            }\n          } else {\n            // Inline mode: stream image scripts to output file\n            streamImageScriptsToFile(reportInfo.reportFilePath, outputFilePath);\n          }\n\n          // Extract all dump scripts from the source report.\n          // After the per-execution append refactor, a single source report\n          // may contain multiple <script type=\"midscene_web_dump\"> tags\n          // (one per execution). We merge them into a single ReportActionDump.\n          // Filter by data-group-id to exclude false matches from the template's\n          // bundled JS code, which also references the midscene_web_dump type string.\n          const allDumps = extractAllDumpScriptsSync(\n            reportInfo.reportFilePath,\n          ).filter((d) => d.openTag.includes('data-group-id'));\n          const groupIdMatch = allDumps[0]?.openTag.match(\n            /data-group-id=\"([^\"]+)\"/,\n          );\n          if (groupIdMatch) {\n            mergedGroupId = decodeURIComponent(groupIdMatch[1]);\n          }\n          const extractedDumpString =\n            allDumps.length > 0\n              ? this.mergeDumpScripts(allDumps.map((d) => d.content))\n              : extractLastDumpScriptSync(reportInfo.reportFilePath);\n          if (extractedDumpString) {\n            dumpString = extractedDumpString;\n          }\n        }\n\n        const agentReport = tryParseAgentReportDump(dumpString);\n        if (agentReport) {\n          agentReports.push(agentReport);\n        }\n\n        const reportHtmlStr = `${reportHTMLContent(\n          {\n            dumpString,\n            attributes: {\n              'data-group-id': mergedGroupId,\n              [DATA_SCREENSHOT_MODE_ATTR]: hasDirectoryModeReport\n                ? 'directory'\n                : 'inline',\n              playwright_test_duration: reportAttributes.testDuration,\n              playwright_test_status: reportAttributes.testStatus,\n              playwright_test_title: reportAttributes.testTitle,\n              playwright_test_id: reportAttributes.testId,\n              playwright_test_description: reportAttributes.testDescription,\n              is_merged: true,\n            },\n          },\n          undefined,\n          undefined,\n          false,\n        )}\\n`;\n\n        appendFileSync(outputFilePath, reportHtmlStr);\n      }\n\n      const agentComment = mergedAgentReportComment(agentReports);\n      if (agentComment) {\n        appendFileSync(outputFilePath, agentComment);\n      }\n\n      // Close the HTML document\n      appendFileSync(outputFilePath, `${htmlEndTag}\\n`);\n\n      logMsg(`Successfully merged new report: ${outputFilePath}`);\n\n      // Remove original reports if needed\n      if (rmOriginalReports) {\n        for (const info of this.reportInfos) {\n          if (!info.reportFilePath) continue;\n          try {\n            if (isDirectoryBasedReport(info.reportFilePath)) {\n              // The report owns its directory (`{name}/index.html`) — remove the\n              // whole folder, whether screenshots are external or inlined.\n              const reportDir = path.dirname(info.reportFilePath);\n              rmSync(reportDir, { recursive: true, force: true });\n            } else {\n              unlinkSync(info.reportFilePath);\n            }\n          } catch (error) {\n            logMsg(`Error deleting report ${info.reportFilePath}: ${error}`);\n          }\n        }\n        logMsg(`Removed ${this.reportInfos.length} original reports`);\n      }\n      return outputFilePath;\n    } catch (error) {\n      logMsg(`Error in mergeReports: ${error}`);\n      throw error;\n    }\n  }\n}\n\nexport interface SplitReportHtmlOptions {\n  htmlPath: string;\n  outputDir: string;\n}\n\nexport interface SplitReportHtmlResult {\n  executionJsonFiles: string[];\n  screenshotFiles: string[];\n}\n\nexport interface CollectedReportExecutions {\n  baseDump: ReportActionDump;\n  executions: IExecutionDump[];\n}\n\n/**\n * Collect executions from a report HTML, deduplicating by stable id while\n * keeping only the latest occurrence. Old-format executions without id are\n * always preserved.\n */\nexport function collectDedupedExecutions(\n  htmlPath: string,\n): CollectedReportExecutions {\n  let baseDump: ReportActionDump | null = null;\n  let executionSerial = 0;\n  const latestSerialByExecutionId = new Map<string, number>();\n\n  streamDumpScriptsSync(htmlPath, (dumpScript) => {\n    if (!dumpScript.openTag.includes('data-group-id')) {\n      return false;\n    }\n    const groupedDump = ReportActionDump.fromSerializedString(\n      antiEscapeScriptTag(dumpScript.content),\n    );\n    for (const execution of groupedDump.executions) {\n      executionSerial += 1;\n      if (execution.id) {\n        latestSerialByExecutionId.set(execution.id, executionSerial);\n      }\n    }\n    return false;\n  });\n\n  const executions: IExecutionDump[] = [];\n  executionSerial = 0;\n  streamDumpScriptsSync(htmlPath, (dumpScript) => {\n    if (!dumpScript.openTag.includes('data-group-id')) {\n      return false;\n    }\n\n    const groupedDump = ReportActionDump.fromSerializedString(\n      antiEscapeScriptTag(dumpScript.content),\n    );\n    if (!baseDump) {\n      baseDump = groupedDump;\n    }\n\n    for (const execution of groupedDump.executions) {\n      executionSerial += 1;\n      if (\n        execution.id &&\n        latestSerialByExecutionId.get(execution.id) !== executionSerial\n      ) {\n        continue;\n      }\n      executions.push(execution);\n    }\n\n    return false;\n  });\n\n  if (!baseDump) {\n    throw new Error(`No report dump scripts found in ${htmlPath}`);\n  }\n\n  return {\n    baseDump,\n    executions,\n  };\n}\n\nfunction extensionByMimeType(mimeType: string): 'png' | 'jpeg' {\n  if (mimeType === 'image/png') return 'png';\n  if (mimeType === 'image/jpeg') return 'jpeg';\n  throw new Error(`Unsupported screenshot mime type: ${mimeType}`);\n}\n\nfunction externalizeScreenshotsInExecution(\n  execution: IExecutionDump,\n  opts: {\n    htmlPath: string;\n    screenshotsDir: string;\n    writtenFiles: Set<string>;\n  },\n): void {\n  const visit = (node: unknown): void => {\n    if (Array.isArray(node)) {\n      for (const item of node) {\n        visit(item);\n      }\n      return;\n    }\n\n    if (typeof node !== 'object' || node === null) return;\n\n    const ref = normalizeScreenshotRef(node);\n    if (ref) {\n      const ext = extensionByMimeType(ref.mimeType);\n      const fileName = `${ref.id}.${ext}`;\n      const relativePath = `./screenshots/${fileName}`;\n      const absolutePath = path.join(opts.screenshotsDir, fileName);\n\n      if (!opts.writtenFiles.has(fileName)) {\n        const resolved = resolveScreenshotSource(ref, {\n          reportPath: opts.htmlPath,\n        });\n        if (resolved.type === 'data-uri') {\n          const rawBase64 = resolved.dataUri.replace(\n            /^data:image\\/[a-zA-Z+]+;base64,/,\n            '',\n          );\n          writeFileSync(absolutePath, Buffer.from(rawBase64, 'base64'));\n        } else {\n          copyFileSync(resolved.filePath, absolutePath);\n        }\n        opts.writtenFiles.add(fileName);\n      }\n\n      ref.storage = 'file';\n      ref.path = relativePath;\n      return;\n    }\n\n    for (const value of Object.values(node)) {\n      visit(value);\n    }\n  };\n\n  visit(execution);\n}\n\n/**\n * Reverse parse a Midscene report HTML into per-execution JSON files and\n * externalized screenshots.\n */\nexport function splitReportHtmlByExecution(\n  options: SplitReportHtmlOptions,\n): SplitReportHtmlResult {\n  const { htmlPath, outputDir } = options;\n  const screenshotsDir = path.join(outputDir, 'screenshots');\n\n  mkdirSync(outputDir, { recursive: true });\n  mkdirSync(screenshotsDir, { recursive: true });\n\n  const executionJsonFiles: string[] = [];\n  const writtenScreenshotFiles = new Set<string>();\n  const { baseDump, executions } = collectDedupedExecutions(htmlPath);\n\n  let fileIndex = 0;\n  for (const execution of executions) {\n    fileIndex += 1;\n    externalizeScreenshotsInExecution(execution, {\n      htmlPath,\n      screenshotsDir,\n      writtenFiles: writtenScreenshotFiles,\n    });\n    const singleExecutionDump = new ReportActionDump({\n      sdkVersion: baseDump.sdkVersion,\n      groupName: baseDump.groupName,\n      groupDescription: baseDump.groupDescription,\n      modelBriefs: baseDump.modelBriefs,\n      deviceType: baseDump.deviceType,\n      executions: [execution],\n    });\n\n    const jsonFilePath = path.join(outputDir, `${fileIndex}.execution.json`);\n    writeFileSync(jsonFilePath, singleExecutionDump.serialize(2), 'utf-8');\n    executionJsonFiles.push(jsonFilePath);\n  }\n\n  return {\n    executionJsonFiles,\n    screenshotFiles: Array.from(writtenScreenshotFiles)\n      .sort()\n      .map((fileName) => path.join(screenshotsDir, fileName)),\n  };\n}\n"],"names":["screenshotModeAttrRegExp","RegExp","DATA_SCREENSHOT_MODE_ATTR","readDeclaredScreenshotMode","reportFilePath","mode","streamDumpScriptsSync","openTag","match","isDirectoryModeReport","path","declared","existsSync","isDirectoryBasedReport","dedupeExecutionsKeepLatest","executions","noIdCounter","deduped","Map","exec","key","Array","peekReportSdkVersion","dump","extractLastDumpScriptSync","warnedMismatchedVersions","Set","tryParseAgentReportDump","dumpString","trimmed","ReportActionDump","mergedAgentReportComment","reports","generateAgentReportComment","deviceTypes","report","Boolean","mergedReport","getVersion","ReportMergingTool","groupName","groupDescription","reportInfo","sourceVersion","currentVersion","logMsg","contents","unescaped","c","antiEscapeScriptTag","base","allExecutions","i","other","reportFileName","opts","rmOriginalReports","overwrite","outputDir","targetDir","getMidsceneRunSubDir","mkdirSync","isDirModeByIndex","info","hasDirectoryModeReport","resolvedName","getReportFileName","outputFilePath","Error","rmSync","unlinkSync","htmlEndTag","tpl","getReportTpl","htmlEndIdx","tplWithoutClose","appendFileSync","getBaseUrlFixScript","agentReports","reportAttributes","mergedGroupId","reportDir","screenshotsDir","mergedScreenshotsDir","file","readdirSync","src","dest","copyFileSync","streamImageScriptsToFile","allDumps","extractAllDumpScriptsSync","d","groupIdMatch","decodeURIComponent","extractedDumpString","agentReport","reportHtmlStr","reportHTMLContent","undefined","agentComment","error","collectDedupedExecutions","htmlPath","baseDump","executionSerial","latestSerialByExecutionId","dumpScript","groupedDump","execution","extensionByMimeType","mimeType","externalizeScreenshotsInExecution","visit","node","item","ref","normalizeScreenshotRef","ext","fileName","relativePath","absolutePath","resolved","resolveScreenshotSource","rawBase64","writeFileSync","Buffer","value","Object","splitReportHtmlByExecution","options","executionJsonFiles","writtenScreenshotFiles","fileIndex","singleExecutionDump","jsonFilePath"],"mappings":";;;;;;;;;;;;;;;;;;;AA+CA,MAAMA,2BAA2B,IAAIC,OACnC,GAAGC,0BAA0B,qBAAqB,CAAC;AAGrD,SAASC,2BACPC,cAAsB;IAEtB,IAAIC;IACJ,IAAI;QACFC,sBAAsBF,gBAAgB,CAAC,EAAEG,OAAO,EAAE;YAEhD,IAAI,CAACA,QAAQ,QAAQ,CAAC,kBAAkB,OAAO;YAC/C,MAAMC,QAAQD,QAAQ,KAAK,CAACP;YAC5BK,OAAOG,OAAO,CAAC,EAAE;YACjB,OAAO;QACT;IACF,EAAE,OAAM,CAER;IACA,OAAOH;AACT;AAWO,SAASI,sBAAsBL,cAAsB;IAI1D,IAAIM,AAAkC,iBAAlCA,SAAcN,iBAAkC,OAAO;IAE3D,MAAMO,WAAWR,2BAA2BC;IAC5C,IAAIO,UAAU,OAAOA,AAAa,gBAAbA;IAGrB,OAAOC,WAAWF,KAAUA,QAAaN,iBAAiB;AAC5D;AAaA,SAASS,uBAAuBT,cAAsB;IACpD,OAAOM,AAAkC,iBAAlCA,SAAcN;AACvB;AAMO,SAASU,2BACdC,UAAe;IAEf,IAAIC,cAAc;IAClB,MAAMC,UAAU,IAAIC;IACpB,KAAK,MAAMC,QAAQJ,WAAY;QAC7B,MAAMK,MAAMD,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAEH,eAAe;QACjDC,QAAQ,GAAG,CAACG,KAAKD;IACnB;IACA,OAAOE,MAAM,IAAI,CAACJ,QAAQ,MAAM;AAClC;AAMA,SAASK,qBAAqBlB,cAAsB;IAClD,IAAI;QACF,MAAMmB,OAAOC,0BAA0BpB;QACvC,IAAI,CAACmB,MAAM;QACX,MAAMf,QAAQe,KAAK,KAAK,CAAC;QACzB,OAAOf,OAAO,CAAC,EAAE;IACnB,EAAE,OAAM;QACN;IACF;AACF;AAEA,MAAMiB,2BAA2B,IAAIC;AAErC,SAASC,wBAAwBC,UAAkB;IACjD,MAAMC,UAAUD,WAAW,SAAS;IACpC,IAAI,CAACC,QAAQ,UAAU,CAAC,QAAQ,CAACA,QAAQ,QAAQ,CAAC,iBAChD,OAAO;IAGT,IAAI;QACF,OAAOC,iBAAiB,oBAAoB,CAACD;IAC/C,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,SAASE,yBAAyBC,OAA2B;IAC3D,IAAIA,AAAmB,MAAnBA,QAAQ,MAAM,EAChB,OAAO;IAET,IAAIA,AAAmB,MAAnBA,QAAQ,MAAM,EAChB,OAAOC,2BAA2BD,OAAO,CAAC,EAAE;IAG9C,MAAME,cAAcb,MAAM,IAAI,CAC5B,IAAIK,IAAIM,QAAQ,GAAG,CAAC,CAACG,SAAWA,OAAO,UAAU,EAAE,MAAM,CAACC;IAE5D,MAAMC,eAAe,IAAIP,iBAAiB;QACxC,YAAYE,OAAO,CAAC,EAAE,CAAC,UAAU,IAAIM;QACrC,WAAW;QACX,kBAAkB;QAClB,aAAaN,QAAQ,OAAO,CAAC,CAACG,SAAWA,OAAO,WAAW,IAAI,EAAE;QACjE,YAAYD,AAAuB,MAAvBA,YAAY,MAAM,GAASA,WAAW,CAAC,EAAE,GAAG;QACxD,YAAYF,QAAQ,OAAO,CAAC,CAACG,SAAWA,OAAO,UAAU,IAAI,EAAE;IACjE;IACA,OAAOF,2BAA2BI;AACpC;AAEO,MAAME;IAGH,sBAAsBC,SAAiB,EAAEC,gBAAyB,EAAE;QAC1E,OAAO,IAAIX,iBAAiB;YAC1B,YAAY;YACZU;YACAC;YACA,aAAa,EAAE;YACf,YAAY,EAAE;QAChB,GAAG,SAAS;IACd;IAEO,OAAOC,UAAoC,EAAE;QAClD,IAAIA,WAAW,cAAc,EAAE;YAC7B,MAAMC,gBAAgBrB,qBAAqBoB,WAAW,cAAc;YACpE,MAAME,iBAAiBN;YACvB,IACEK,iBACAC,kBACAD,kBAAkBC,kBAClB,CAACnB,yBAAyB,GAAG,CAACkB,gBAC9B;gBACAlB,yBAAyB,GAAG,CAACkB;gBAC7BE,OACE,CAAC,iGAAiG,EAAEF,cAAc,kCAAkC,EAAEC,eAAe,gRAAgR,CAAC;YAE1b;QACF;QACA,IAAI,CAAC,WAAW,CAAC,IAAI,CAACF;IACxB;IACO,QAAQ;QACb,IAAI,CAAC,WAAW,GAAG,EAAE;IACvB;IAQQ,iBAAiBI,QAAkB,EAAU;QACnD,MAAMC,YAAYD,SACf,GAAG,CAAC,CAACE,IAAMC,oBAAoBD,IAC/B,MAAM,CAAC,CAACA,IAAMA,EAAE,MAAM,GAAG;QAC5B,IAAID,AAAqB,MAArBA,UAAU,MAAM,EAAQ,OAAO;QACnC,IAAIA,AAAqB,MAArBA,UAAU,MAAM,EAAQ,OAAOA,SAAS,CAAC,EAAE;QAK/C,MAAMG,OAAOpB,iBAAiB,oBAAoB,CAACiB,SAAS,CAAC,EAAE;QAC/D,MAAMI,gBAAgB;eAAID,KAAK,UAAU;SAAC;QAC1C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,UAAU,MAAM,EAAEK,IAAK;YACzC,MAAMC,QAAQvB,iBAAiB,oBAAoB,CAACiB,SAAS,CAACK,EAAE;YAChED,cAAc,IAAI,IAAIE,MAAM,UAAU;QACxC;QACAH,KAAK,UAAU,GAAGpC,2BAA2BqC;QAC7C,OAAOD,KAAK,SAAS;IACvB;IAEO,aACLI,iBAAkC,MAAM,EACxCC,IAIC,EACc;QACf,MAAM,EACJC,oBAAoB,KAAK,EACzBC,YAAY,KAAK,EACjBC,SAAS,EACV,GAAGH,QAAQ,CAAC;QAEb,IAAI,AAA4B,MAA5B,IAAI,CAAC,WAAW,CAAC,MAAM,EAAQ;YACjCV,OAAO;YACP,OAAO;QACT;QAEA,MAAMc,YAAYD,YACdhD,QAAagD,aACbE,qBAAqB;QACzB,IAAIF,WACFG,UAAUF,WAAW;YAAE,WAAW;QAAK;QAOzC,MAAMG,mBAAmB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAACC,OAC7C3B,QACE2B,KAAK,cAAc,IAAItD,sBAAsBsD,KAAK,cAAc;QAGpE,MAAMC,yBAAyBF,iBAAiB,IAAI,CAAC1B;QAErD,MAAM6B,eACJX,AAAmB,WAAnBA,iBACIY,kBAAkB,mBAClBZ;QAIN,MAAMa,iBAAiBH,yBACnBtD,QAAaiD,WAAWM,cAAc,gBACtCvD,QAAaiD,WAAW,GAAGM,aAAa,KAAK,CAAC;QAElD,IAAIX,AAAmB,WAAnBA,kBAA6B1C,WAAWuD,iBAAiB;YAC3D,IAAI,CAACV,WACH,MAAM,IAAIW,MACR,CAAC,4BAA4B,EAAED,eAAe,+CAA+C,CAAC;YAGlG,IAAIH,wBACFK,OAAO3D,QAAayD,iBAAiB;gBAAE,WAAW;gBAAM,OAAO;YAAK;iBAEpEG,WAAWH;QAEf;QAEA,IAAIH,wBACFH,UAAUnD,QAAayD,iBAAiB;YAAE,WAAW;QAAK;QAG5DtB,OACE,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,sCAAsC,CAAC;QAGlF,IAAI;YAGF,MAAM0B,aAAa;YACnB,MAAMC,MAAMC;YACZ,MAAMC,aAAaF,IAAI,WAAW,CAACD;YACnC,MAAMI,kBACJD,AAAe,OAAfA,aAAoBF,IAAI,KAAK,CAAC,GAAGE,cAAcF;YACjDI,eAAeT,gBAAgBQ;YAG/B,IAAIX,wBACFY,eAAeT,gBAAgBU;YAGjC,MAAMC,eAAmC,EAAE;YAG3C,IAAK,IAAI1B,IAAI,GAAGA,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAEA,IAAK;gBAChD,MAAMV,aAAa,IAAI,CAAC,WAAW,CAACU,EAAE;gBACtCP,OAAO,CAAC,kBAAkB,EAAEO,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;gBAE9D,MAAM,EAAE2B,gBAAgB,EAAE,GAAGrC;gBAC7B,IAAId,aAAa,IAAI,CAAC,qBAAqB,CACzCmD,iBAAiB,SAAS,EAC1BA,iBAAiB,eAAe;gBAElC,IAAIC,gBAAgB,CAAC,aAAa,EAAE5B,GAAG;gBAEvC,IAAIV,WAAW,cAAc,EAAE;oBAC7B,IAAIoB,gBAAgB,CAACV,EAAE,EAAE;wBAIvB,MAAM6B,YAAYvE,QAAagC,WAAW,cAAc;wBACxD,MAAMwC,iBAAiBxE,KAAUuE,WAAW;wBAC5C,IAAIrE,WAAWsE,iBAAiB;4BAC9B,MAAMC,uBAAuBzE,KAC3BA,QAAayD,iBACb;4BAEFN,UAAUsB,sBAAsB;gCAAE,WAAW;4BAAK;4BAClD,KAAK,MAAMC,QAAQC,YAAYH,gBAAiB;gCAC9C,MAAMI,MAAM5E,KAAUwE,gBAAgBE;gCACtC,MAAMG,OAAO7E,KAAUyE,sBAAsBC;gCAC7CI,aAAaF,KAAKC;4BACpB;wBACF;oBACF,OAEEE,yBAAyB/C,WAAW,cAAc,EAAEyB;oBAStD,MAAMuB,WAAWC,0BACfjD,WAAW,cAAc,EACzB,MAAM,CAAC,CAACkD,IAAMA,EAAE,OAAO,CAAC,QAAQ,CAAC;oBACnC,MAAMC,eAAeH,QAAQ,CAAC,EAAE,EAAE,QAAQ,MACxC;oBAEF,IAAIG,cACFb,gBAAgBc,mBAAmBD,YAAY,CAAC,EAAE;oBAEpD,MAAME,sBACJL,SAAS,MAAM,GAAG,IACd,IAAI,CAAC,gBAAgB,CAACA,SAAS,GAAG,CAAC,CAACE,IAAMA,EAAE,OAAO,KACnDpE,0BAA0BkB,WAAW,cAAc;oBACzD,IAAIqD,qBACFnE,aAAamE;gBAEjB;gBAEA,MAAMC,cAAcrE,wBAAwBC;gBAC5C,IAAIoE,aACFlB,aAAa,IAAI,CAACkB;gBAGpB,MAAMC,gBAAgB,GAAGC,kBACvB;oBACEtE;oBACA,YAAY;wBACV,iBAAiBoD;wBACjB,CAAC9E,0BAA0B,EAAE8D,yBACzB,cACA;wBACJ,0BAA0Be,iBAAiB,YAAY;wBACvD,wBAAwBA,iBAAiB,UAAU;wBACnD,uBAAuBA,iBAAiB,SAAS;wBACjD,oBAAoBA,iBAAiB,MAAM;wBAC3C,6BAA6BA,iBAAiB,eAAe;wBAC7D,WAAW;oBACb;gBACF,GACAoB,QACAA,QACA,OACA,EAAE,CAAC;gBAELvB,eAAeT,gBAAgB8B;YACjC;YAEA,MAAMG,eAAerE,yBAAyB+C;YAC9C,IAAIsB,cACFxB,eAAeT,gBAAgBiC;YAIjCxB,eAAeT,gBAAgB,GAAGI,WAAW,EAAE,CAAC;YAEhD1B,OAAO,CAAC,gCAAgC,EAAEsB,gBAAgB;YAG1D,IAAIX,mBAAmB;gBACrB,KAAK,MAAMO,QAAQ,IAAI,CAAC,WAAW,CACjC,IAAKA,KAAK,cAAc,EACxB,IAAI;oBACF,IAAIlD,uBAAuBkD,KAAK,cAAc,GAAG;wBAG/C,MAAMkB,YAAYvE,QAAaqD,KAAK,cAAc;wBAClDM,OAAOY,WAAW;4BAAE,WAAW;4BAAM,OAAO;wBAAK;oBACnD,OACEX,WAAWP,KAAK,cAAc;gBAElC,EAAE,OAAOsC,OAAO;oBACdxD,OAAO,CAAC,sBAAsB,EAAEkB,KAAK,cAAc,CAAC,EAAE,EAAEsC,OAAO;gBACjE;gBAEFxD,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,iBAAiB,CAAC;YAC9D;YACA,OAAOsB;QACT,EAAE,OAAOkC,OAAO;YACdxD,OAAO,CAAC,uBAAuB,EAAEwD,OAAO;YACxC,MAAMA;QACR;IACF;;QA7QA,uBAAQ,eAA0C,EAAE;;AA8QtD;AAsBO,SAASC,yBACdC,QAAgB;IAEhB,IAAIC,WAAoC;IACxC,IAAIC,kBAAkB;IACtB,MAAMC,4BAA4B,IAAIxF;IAEtCZ,sBAAsBiG,UAAU,CAACI;QAC/B,IAAI,CAACA,WAAW,OAAO,CAAC,QAAQ,CAAC,kBAC/B,OAAO;QAET,MAAMC,cAAc9E,iBAAiB,oBAAoB,CACvDmB,oBAAoB0D,WAAW,OAAO;QAExC,KAAK,MAAME,aAAaD,YAAY,UAAU,CAAE;YAC9CH,mBAAmB;YACnB,IAAII,UAAU,EAAE,EACdH,0BAA0B,GAAG,CAACG,UAAU,EAAE,EAAEJ;QAEhD;QACA,OAAO;IACT;IAEA,MAAM1F,aAA+B,EAAE;IACvC0F,kBAAkB;IAClBnG,sBAAsBiG,UAAU,CAACI;QAC/B,IAAI,CAACA,WAAW,OAAO,CAAC,QAAQ,CAAC,kBAC/B,OAAO;QAGT,MAAMC,cAAc9E,iBAAiB,oBAAoB,CACvDmB,oBAAoB0D,WAAW,OAAO;QAExC,IAAI,CAACH,UACHA,WAAWI;QAGb,KAAK,MAAMC,aAAaD,YAAY,UAAU,CAAE;YAC9CH,mBAAmB;YACnB,IACEI,CAAAA,UAAU,EAAE,IACZH,0BAA0B,GAAG,CAACG,UAAU,EAAE,MAAMJ,iBAIlD1F,WAAW,IAAI,CAAC8F;QAClB;QAEA,OAAO;IACT;IAEA,IAAI,CAACL,UACH,MAAM,IAAIpC,MAAM,CAAC,gCAAgC,EAAEmC,UAAU;IAG/D,OAAO;QACLC;QACAzF;IACF;AACF;AAEA,SAAS+F,oBAAoBC,QAAgB;IAC3C,IAAIA,AAAa,gBAAbA,UAA0B,OAAO;IACrC,IAAIA,AAAa,iBAAbA,UAA2B,OAAO;IACtC,MAAM,IAAI3C,MAAM,CAAC,kCAAkC,EAAE2C,UAAU;AACjE;AAEA,SAASC,kCACPH,SAAyB,EACzBtD,IAIC;IAED,MAAM0D,QAAQ,CAACC;QACb,IAAI7F,MAAM,OAAO,CAAC6F,OAAO;YACvB,KAAK,MAAMC,QAAQD,KACjBD,MAAME;YAER;QACF;QAEA,IAAI,AAAgB,YAAhB,OAAOD,QAAqBA,AAAS,SAATA,MAAe;QAE/C,MAAME,MAAMC,uBAAuBH;QACnC,IAAIE,KAAK;YACP,MAAME,MAAMR,oBAAoBM,IAAI,QAAQ;YAC5C,MAAMG,WAAW,GAAGH,IAAI,EAAE,CAAC,CAAC,EAAEE,KAAK;YACnC,MAAME,eAAe,CAAC,cAAc,EAAED,UAAU;YAChD,MAAME,eAAe/G,KAAU6C,KAAK,cAAc,EAAEgE;YAEpD,IAAI,CAAChE,KAAK,YAAY,CAAC,GAAG,CAACgE,WAAW;gBACpC,MAAMG,WAAWC,wBAAwBP,KAAK;oBAC5C,YAAY7D,KAAK,QAAQ;gBAC3B;gBACA,IAAImE,AAAkB,eAAlBA,SAAS,IAAI,EAAiB;oBAChC,MAAME,YAAYF,SAAS,OAAO,CAAC,OAAO,CACxC,mCACA;oBAEFG,cAAcJ,cAAcK,OAAO,IAAI,CAACF,WAAW;gBACrD,OACEpC,aAAakC,SAAS,QAAQ,EAAED;gBAElClE,KAAK,YAAY,CAAC,GAAG,CAACgE;YACxB;YAEAH,IAAI,OAAO,GAAG;YACdA,IAAI,IAAI,GAAGI;YACX;QACF;QAEA,KAAK,MAAMO,SAASC,OAAO,MAAM,CAACd,MAChCD,MAAMc;IAEV;IAEAd,MAAMJ;AACR;AAMO,SAASoB,2BACdC,OAA+B;IAE/B,MAAM,EAAE3B,QAAQ,EAAE7C,SAAS,EAAE,GAAGwE;IAChC,MAAMhD,iBAAiBxE,KAAUgD,WAAW;IAE5CG,UAAUH,WAAW;QAAE,WAAW;IAAK;IACvCG,UAAUqB,gBAAgB;QAAE,WAAW;IAAK;IAE5C,MAAMiD,qBAA+B,EAAE;IACvC,MAAMC,yBAAyB,IAAI1G;IACnC,MAAM,EAAE8E,QAAQ,EAAEzF,UAAU,EAAE,GAAGuF,yBAAyBC;IAE1D,IAAI8B,YAAY;IAChB,KAAK,MAAMxB,aAAa9F,WAAY;QAClCsH,aAAa;QACbrB,kCAAkCH,WAAW;YAC3CN;YACArB;YACA,cAAckD;QAChB;QACA,MAAME,sBAAsB,IAAIxG,iBAAiB;YAC/C,YAAY0E,SAAS,UAAU;YAC/B,WAAWA,SAAS,SAAS;YAC7B,kBAAkBA,SAAS,gBAAgB;YAC3C,aAAaA,SAAS,WAAW;YACjC,YAAYA,SAAS,UAAU;YAC/B,YAAY;gBAACK;aAAU;QACzB;QAEA,MAAM0B,eAAe7H,KAAUgD,WAAW,GAAG2E,UAAU,eAAe,CAAC;QACvER,cAAcU,cAAcD,oBAAoB,SAAS,CAAC,IAAI;QAC9DH,mBAAmB,IAAI,CAACI;IAC1B;IAEA,OAAO;QACLJ;QACA,iBAAiB9G,MAAM,IAAI,CAAC+G,wBACzB,IAAI,GACJ,GAAG,CAAC,CAACb,WAAa7G,KAAUwE,gBAAgBqC;IACjD;AACF"}

package/dist/es/service/index.mjs

@@ -46,9 +46,9 @@ class Service {
         const taskInfo = {
             ...this.taskInfo ? this.taskInfo : {},
             durationMs: timeCost,
-            rawResponse: JSON.stringify(rawResponse),
+            rawResponse,
             rawChoiceMessage,
-            formatResponse: JSON.stringify(parseResult),
+            formatResponse: parseResult,
             usage,
             searchArea: searchArea.trace.sourceRect,
             searchAreaRawResponse: searchArea.trace.rawResponse,
@@ -212,7 +212,7 @@ class Service {
             durationMs: timeCost,
             rawResponse,
             rawChoiceMessage,
-            formatResponse: JSON.stringify(parseResult),
+            formatResponse: parseResult,
             usage,
             reasoning_content
         };

package/dist/es/service/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"service/index.mjs","sources":["../../../src/service/index.ts"],"sourcesContent":["import { defaultModelFamilyRequiredForLocateMessage } from '@/ai-model/errors';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  AiLocateSection,\n  buildSearchAreaConfig,\n} from '@/ai-model/inspect';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport {\n  AIResponseParseError,\n  callAIWithObjectResponse,\n} from '@/ai-model/service-caller';\nimport type { AIArgs } from '@/ai-model/types';\nimport type { SearchAreaConfig } from '@/ai-model/workflows/inspect/types';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  PlanningLocateParam,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport {\n  compositeElementInfoImg,\n  compositePointMarkerImg,\n  cropByRect,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionContentPart } from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  createServiceDump,\n  getDescribeDeepContextAreas,\n  getDescribeDeepLocateResizeSize,\n  getDescribeMarkerBorderThickness,\n  getDescribeMarkerRect,\n  getRectInCrop,\n  recoverDescribeResponseFromParseError,\n} from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\ninterface LocateSearchAreaResult {\n  config?: SearchAreaConfig;\n  trace: {\n    sourceRect?: Rect;\n    rawResponse?: string;\n    rawChoiceMessage?: unknown;\n    usage?: AIUsageInfo;\n  };\n}\n\nconst debug = getDebug('ai:service');\n\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: PlanningLocateParam,\n    opt: LocateOpts,\n    modelRuntime: ModelRuntime,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const { config: modelConfig } = modelRuntime;\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    if (!modelConfig.modelFamily) {\n      throw new Error(defaultModelFamilyRequiredForLocateMessage);\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    const searchArea = await this.resolveLocateSearchArea({\n      query,\n      queryPrompt,\n      opt,\n      context,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content,\n    } = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchArea.config,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      rawChoiceMessage,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea: searchArea.trace.sourceRect,\n      searchAreaRawResponse: searchArea.trace.rawResponse,\n      searchAreaRawChoiceMessage: searchArea.trace.rawChoiceMessage,\n      searchAreaUsage: searchArea.trace.usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea.trace.sourceRect,\n      error: errorLog,\n    };\n\n    const element = parseResult.element;\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: element ? [element] : [],\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (element) {\n      return {\n        element: {\n          center: element.center,\n          rect: element.rect,\n          description: element.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  private async resolveLocateSearchArea(options: {\n    query: PlanningLocateParam;\n    queryPrompt: TUserPrompt;\n    opt: LocateOpts;\n    context: UIContext;\n    modelRuntime: ModelRuntime;\n    abortSignal?: AbortSignal;\n  }): Promise<LocateSearchAreaResult> {\n    const { query, queryPrompt, opt, context, modelRuntime, abortSignal } =\n      options;\n    const { adapter } = modelRuntime;\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n    if (!query.deepLocate) {\n      return { trace: {} };\n    }\n\n    if (hasPlanLocatedElement) {\n      const config = await buildSearchAreaConfig({\n        context,\n        baseRect: opt.planLocatedElement!.rect,\n      });\n\n      return {\n        config,\n        trace: {\n          sourceRect: config.sourceRect,\n          rawResponse: JSON.stringify({\n            source: 'plan-located-element',\n            rect: opt.planLocatedElement!.rect,\n          }),\n        },\n      };\n    }\n\n    if (adapter.locate.supportsSearchArea) {\n      const searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: queryPrompt,\n        modelRuntime,\n        abortSignal,\n      });\n      const { searchAreaConfig } = searchAreaResponse;\n      assert(\n        searchAreaConfig,\n        `cannot find search area for \"${queryPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n\n      return {\n        config: searchAreaConfig,\n        trace: {\n          sourceRect: searchAreaConfig.sourceRect,\n          rawResponse: searchAreaResponse.rawResponse,\n          rawChoiceMessage: searchAreaResponse.rawChoiceMessage,\n          usage: searchAreaResponse.usage,\n        },\n      };\n    }\n\n    const firstPassLocateResult = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      modelRuntime,\n      abortSignal,\n    });\n    assert(\n      firstPassLocateResult.rect,\n      `cannot find search area for \"${queryPrompt}\"${\n        firstPassLocateResult.parseResult.errors?.length\n          ? `: ${firstPassLocateResult.parseResult.errors.join('\\n')}`\n          : ''\n      }`,\n    );\n\n    const config = await buildSearchAreaConfig({\n      context,\n      baseRect: firstPassLocateResult.rect,\n    });\n\n    return {\n      config,\n      trace: {\n        sourceRect: config.sourceRect,\n        rawResponse: JSON.stringify({\n          source: 'deep-locate-first-pass',\n          rect: firstPassLocateResult.rect,\n          rawResponse: firstPassLocateResult.rawResponse,\n        }),\n        rawChoiceMessage: firstPassLocateResult.rawChoiceMessage,\n        usage: firstPassLocateResult.usage,\n      },\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let rawChoiceMessage: unknown;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelRuntime,\n        pageDescription,\n        abortSignal: executionOptions?.abortSignal,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      rawChoiceMessage = result.rawChoiceMessage;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          rawChoiceMessage: error.rawChoiceMessage,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawChoiceMessage,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelRuntime: ModelRuntime,\n    opt?: {\n      deepDescribe?: boolean;\n      context?: UIContext;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = opt?.context || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetFromPoint = Array.isArray(target);\n    const targetRect: Rect = targetFromPoint\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n    const targetPoint = targetFromPoint\n      ? {\n          x: target[0],\n          y: target[1],\n        }\n      : undefined;\n\n    const usePointMarker = targetFromPoint;\n    const imagePayload = usePointMarker\n      ? await compositePointMarkerImg({\n          inputImgBase64: screenshotBase64,\n          size: shotSize,\n          point: targetPoint!,\n        })\n      : await compositeElementInfoImg({\n          inputImgBase64: screenshotBase64,\n          size: shotSize,\n          elementsPositionInfo: [\n            {\n              rect: getDescribeMarkerRect(targetRect),\n            },\n          ],\n          borderThickness: getDescribeMarkerBorderThickness(targetRect),\n          centerPoint: true,\n        });\n\n    const shouldDeepDescribe = opt?.deepDescribe;\n    let imageContent: ChatCompletionContentPart[];\n    if (shouldDeepDescribe) {\n      const contextAreas = getDescribeDeepContextAreas(targetRect, shotSize);\n      const contextImages = await Promise.all(\n        contextAreas.map(async (area) => {\n          debug('describe: cropping deep context area', area);\n          const croppedResult = await cropByRect(screenshotBase64, area.rect);\n          const cropSize = {\n            width: croppedResult.width,\n            height: croppedResult.height,\n          };\n          const targetInCrop = getRectInCrop(targetRect, area.rect, cropSize);\n          const markedCropPayload = targetFromPoint\n            ? await compositePointMarkerImg({\n                inputImgBase64: croppedResult.imageBase64,\n                size: cropSize,\n                point: {\n                  x: targetPoint!.x - area.rect.left,\n                  y: targetPoint!.y - area.rect.top,\n                },\n              })\n            : await compositeElementInfoImg({\n                inputImgBase64: croppedResult.imageBase64,\n                size: cropSize,\n                elementsPositionInfo: [\n                  {\n                    rect: getDescribeMarkerRect(targetInCrop),\n                  },\n                ],\n                borderThickness: getDescribeMarkerBorderThickness(targetInCrop),\n                centerPoint: true,\n              });\n          const resizeSize = getDescribeDeepLocateResizeSize(croppedResult);\n          return {\n            kind: area.kind,\n            imageBase64: resizeSize\n              ? await resizeImgBase64(markedCropPayload, resizeSize)\n              : markedCropPayload,\n          };\n        }),\n      );\n      const contextImageContent =\n        contextImages.flatMap<ChatCompletionContentPart>((item, index) => [\n          {\n            type: 'text',\n            text: `Image ${index + 2}: focused detail crop around the target, for reading text, icon shape, and exact local boundaries.`,\n          },\n          {\n            type: 'image_url',\n            image_url: {\n              url: item.imageBase64,\n              detail: 'high',\n            },\n          },\n        ]);\n\n      imageContent = [\n        {\n          type: 'text' as const,\n          text: 'Use these images together to describe the real UI target marked by the temporary callout. Do not describe the marker itself.',\n        },\n        {\n          type: 'text' as const,\n          text: 'Image 1: full screenshot overview with the target marker, for page position and ownership context.',\n        },\n        {\n          type: 'image_url' as const,\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        ...contextImageContent,\n      ];\n    } else {\n      imageContent = [\n        {\n          type: 'text' as const,\n          text: 'Full screenshot with a temporary callout marking the target:',\n        },\n        {\n          type: 'image_url' as const,\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ];\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: imageContent,\n      },\n    ];\n\n    let res: Awaited<\n      ReturnType<typeof callAIWithObjectResponse<AIDescribeElementResponse>>\n    >;\n    try {\n      res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n        msgs,\n        modelRuntime,\n      );\n    } catch (error) {\n      const recoveredResponse = recoverDescribeResponseFromParseError(error);\n      if (!recoveredResponse) {\n        throw error;\n      }\n      debug('describe: recovered malformed description JSON response');\n      return recoveredResponse;\n    }\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Service","query","opt","modelRuntime","abortSignal","modelConfig","queryPrompt","assert","Error","defaultModelFamilyRequiredForLocateMessage","context","searchArea","startTime","Date","parseResult","rect","rawResponse","rawChoiceMessage","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","element","dump","createServiceDump","ServiceError","options","adapter","hasPlanLocatedElement","config","buildSearchAreaConfig","searchAreaResponse","AiLocateSection","searchAreaConfig","firstPassLocateResult","dataDemand","pageDescription","multimodalPrompt","executionOptions","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetFromPoint","Array","targetRect","Math","targetPoint","undefined","usePointMarker","imagePayload","compositePointMarkerImg","compositeElementInfoImg","getDescribeMarkerRect","getDescribeMarkerBorderThickness","shouldDeepDescribe","imageContent","contextAreas","getDescribeDeepContextAreas","contextImages","Promise","area","croppedResult","cropByRect","cropSize","targetInCrop","getRectInCrop","markedCropPayload","resizeSize","getDescribeDeepLocateResizeSize","resizeImgBase64","contextImageContent","item","index","msgs","res","callAIWithObjectResponse","recoveredResponse","recoverDescribeResponseFromParseError","content"],"mappings":";;;;;;;;;;;;;;;;;;;AA0EA,MAAMA,QAAQC,SAAS;AAER,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,YAA0B,EAC1BC,WAAyB,EACM;QAC/B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGF;QAChC,MAAMG,cAAc,AAAiB,YAAjB,OAAOL,QAAqBA,QAAQA,MAAM,MAAM;QACpEM,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAON,OAAoB;QAElC,IAAI,CAACI,YAAY,WAAW,EAC1B,MAAM,IAAIG,MAAMC;QAGlB,MAAMC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,MAAMS,aAAa,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACpDV;YACAK;YACAJ;YACAQ;YACAP;YACAC;QACF;QAEA,MAAMQ,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,gBAAgB;YACxBV;YACA,0BAA0BJ;YAC1B,cAAcK,WAAW,MAAM;YAC/BR;YACAC;QACF;QAEA,MAAMiB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACP;YAC5BC;YACA,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACA,YAAYP,WAAW,KAAK,CAAC,UAAU;YACvC,uBAAuBA,WAAW,KAAK,CAAC,WAAW;YACnD,4BAA4BA,WAAW,KAAK,CAAC,gBAAgB;YAC7D,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACvCQ;QACF;QAEA,IAAIK;QACJ,IAAIV,YAAY,MAAM,EAAE,QACtBU,WAAW,CAAC,4BAA4B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASnB;YACX;YACA,aAAaS;YACb,MAAM;YACNO;YACA,YAAY,CAAC,CAACX,WAAW,KAAK,CAAC,UAAU;YACzC,OAAOa;QACT;QAEA,MAAME,UAAUZ,YAAY,OAAO;QAEnC,MAAMa,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC,UAAU;gBAACA;aAAQ,GAAG,EAAE;QAC1C;QAEA,IAAIF,UACF,MAAM,IAAIK,aAAaL,UAAUG;QAGnC,IAAID,SACF,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,MAAM;gBACtB,MAAMA,QAAQ,IAAI;gBAClB,aAAaA,QAAQ,WAAW;YAClC;YACAX;YACAY;QACF;QAGF,OAAO;YACL,SAAS;YACTZ;YACAY;QACF;IACF;IAEA,MAAc,wBAAwBG,OAOrC,EAAmC;QAClC,MAAM,EAAE7B,KAAK,EAAEK,WAAW,EAAEJ,GAAG,EAAEQ,OAAO,EAAEP,YAAY,EAAEC,WAAW,EAAE,GACnE0B;QACF,MAAM,EAAEC,OAAO,EAAE,GAAG5B;QACpB,MAAM6B,wBAAwB,CAAC,CAAC9B,KAAK,oBAAoB;QAEzD,IAAI,CAACD,MAAM,UAAU,EACnB,OAAO;YAAE,OAAO,CAAC;QAAE;QAGrB,IAAI+B,uBAAuB;YACzB,MAAMC,SAAS,MAAMC,sBAAsB;gBACzCxB;gBACA,UAAUR,IAAI,kBAAkB,CAAE,IAAI;YACxC;YAEA,OAAO;gBACL+B;gBACA,OAAO;oBACL,YAAYA,OAAO,UAAU;oBAC7B,aAAaV,KAAK,SAAS,CAAC;wBAC1B,QAAQ;wBACR,MAAMrB,IAAI,kBAAkB,CAAE,IAAI;oBACpC;gBACF;YACF;QACF;QAEA,IAAI6B,QAAQ,MAAM,CAAC,kBAAkB,EAAE;YACrC,MAAMI,qBAAqB,MAAMC,gBAAgB;gBAC/C1B;gBACA,oBAAoBJ;gBACpBH;gBACAC;YACF;YACA,MAAM,EAAEiC,gBAAgB,EAAE,GAAGF;YAC7B5B,OACE8B,kBACA,CAAC,6BAA6B,EAAE/B,YAAY,CAAC,EAC3C6B,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAGJ,OAAO;gBACL,QAAQE;gBACR,OAAO;oBACL,YAAYA,iBAAiB,UAAU;oBACvC,aAAaF,mBAAmB,WAAW;oBAC3C,kBAAkBA,mBAAmB,gBAAgB;oBACrD,OAAOA,mBAAmB,KAAK;gBACjC;YACF;QACF;QAEA,MAAMG,wBAAwB,MAAMlB,gBAAgB;YAClDV;YACA,0BAA0BJ;YAC1BH;YACAC;QACF;QACAG,OACE+B,sBAAsB,IAAI,EAC1B,CAAC,6BAA6B,EAAEhC,YAAY,CAAC,EAC3CgC,sBAAsB,WAAW,CAAC,MAAM,EAAE,SACtC,CAAC,EAAE,EAAEA,sBAAsB,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,GAC1D,IACJ;QAGJ,MAAML,SAAS,MAAMC,sBAAsB;YACzCxB;YACA,UAAU4B,sBAAsB,IAAI;QACtC;QAEA,OAAO;YACLL;YACA,OAAO;gBACL,YAAYA,OAAO,UAAU;gBAC7B,aAAaV,KAAK,SAAS,CAAC;oBAC1B,QAAQ;oBACR,MAAMe,sBAAsB,IAAI;oBAChC,aAAaA,sBAAsB,WAAW;gBAChD;gBACA,kBAAkBA,sBAAsB,gBAAgB;gBACxD,OAAOA,sBAAsB,KAAK;YACpC;QACF;IACF;IAEA,MAAM,QACJC,UAA+B,EAC/BpC,YAA0B,EAC1BD,GAA0B,EAC1BsC,eAAwB,EACxBC,gBAAoC,EACpC/B,OAAmB,EACnBgC,gBAEC,EACiC;QAClCnC,OAAOG,SAAS;QAChBH,OACE,AAAsB,YAAtB,OAAOgC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAM3B,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMwB,SAAS,MAAMC,qBAAwB;gBAC3ClC;gBACA,WAAW6B;gBACXE;gBACA,eAAevC;gBACfC;gBACAqC;gBACA,aAAaE,kBAAkB;YACjC;YACA5B,cAAc6B,OAAO,WAAW;YAChC3B,cAAc2B,OAAO,WAAW;YAChC1B,mBAAmB0B,OAAO,gBAAgB;YAC1CzB,QAAQyB,OAAO,KAAK;YACpBxB,oBAAoBwB,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,sBAAsB;gBAEzC,MAAMzB,WAAWR,KAAK,GAAG,KAAKD;gBAC9B,MAAMU,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAawB,MAAM,WAAW;oBAC9B,kBAAkBA,MAAM,gBAAgB;oBACxC,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMlB,OAAOC,kBAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEW;oBAAW;oBACxB,MAAM;oBACNjB;oBACA,OAAOuB,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIhB,aAAagB,MAAM,OAAO,EAAElB;YACxC;YACA,MAAMkB;QACR;QAEA,MAAMxB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZL;YACAC;YACA,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIV,YAAY,MAAM,EAAE,QACtBU,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTc;YACF;YACA,MAAM;YACNjB;YACA,OAAOE;QACT;QAEA,MAAM,EAAEuB,IAAI,EAAEC,OAAO,EAAE,GAAGlC,eAAe,CAAC;QAE1C,MAAMa,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACXsB;QACF;QAEA,IAAIvB,YAAY,CAACuB,MACf,MAAM,IAAIlB,aAAaL,UAAUG;QAGnC,OAAO;YACLoB;YACAC;YACA9B;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJsB,MAA+B,EAC/B9C,YAA0B,EAC1BD,GAGC,EACwD;QACzDK,OAAO0C,QAAQ;QACf,MAAMvC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAC9D,MAAM,EAAEgD,QAAQ,EAAE,GAAGxC;QACrB,MAAMyC,mBAAmBzC,QAAQ,UAAU,CAAC,MAAM;QAClDH,OAAO4C,kBAAkB;QACzB,MAAMC,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,kBAAkBC,MAAM,OAAO,CAACP;QACtC,MAAMQ,aAAmBF,kBACrB;YACE,MAAMG,KAAK,KAAK,CAACT,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKI,KAAK,KAAK,CAACT,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QACJ,MAAMU,cAAcJ,kBAChB;YACE,GAAGN,MAAM,CAAC,EAAE;YACZ,GAAGA,MAAM,CAAC,EAAE;QACd,IACAW;QAEJ,MAAMC,iBAAiBN;QACvB,MAAMO,eAAeD,iBACjB,MAAME,wBAAwB;YAC5B,gBAAgBZ;YAChB,MAAMD;YACN,OAAOS;QACT,KACA,MAAMK,wBAAwB;YAC5B,gBAAgBb;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMe,sBAAsBR;gBAC9B;aACD;YACD,iBAAiBS,iCAAiCT;YAClD,aAAa;QACf;QAEJ,MAAMU,qBAAqBjE,KAAK;QAChC,IAAIkE;QACJ,IAAID,oBAAoB;YACtB,MAAME,eAAeC,4BAA4Bb,YAAYP;YAC7D,MAAMqB,gBAAgB,MAAMC,QAAQ,GAAG,CACrCH,aAAa,GAAG,CAAC,OAAOI;gBACtB3E,MAAM,wCAAwC2E;gBAC9C,MAAMC,gBAAgB,MAAMC,WAAWxB,kBAAkBsB,KAAK,IAAI;gBAClE,MAAMG,WAAW;oBACf,OAAOF,cAAc,KAAK;oBAC1B,QAAQA,cAAc,MAAM;gBAC9B;gBACA,MAAMG,eAAeC,cAAcrB,YAAYgB,KAAK,IAAI,EAAEG;gBAC1D,MAAMG,oBAAoBxB,kBACtB,MAAMQ,wBAAwB;oBAC5B,gBAAgBW,cAAc,WAAW;oBACzC,MAAME;oBACN,OAAO;wBACL,GAAGjB,YAAa,CAAC,GAAGc,KAAK,IAAI,CAAC,IAAI;wBAClC,GAAGd,YAAa,CAAC,GAAGc,KAAK,IAAI,CAAC,GAAG;oBACnC;gBACF,KACA,MAAMT,wBAAwB;oBAC5B,gBAAgBU,cAAc,WAAW;oBACzC,MAAME;oBACN,sBAAsB;wBACpB;4BACE,MAAMX,sBAAsBY;wBAC9B;qBACD;oBACD,iBAAiBX,iCAAiCW;oBAClD,aAAa;gBACf;gBACJ,MAAMG,aAAaC,gCAAgCP;gBACnD,OAAO;oBACL,MAAMD,KAAK,IAAI;oBACf,aAAaO,aACT,MAAME,gBAAgBH,mBAAmBC,cACzCD;gBACN;YACF;YAEF,MAAMI,sBACJZ,cAAc,OAAO,CAA4B,CAACa,MAAMC,QAAU;oBAChE;wBACE,MAAM;wBACN,MAAM,CAAC,MAAM,EAAEA,QAAQ,EAAE,kGAAkG,CAAC;oBAC9H;oBACA;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKD,KAAK,WAAW;4BACrB,QAAQ;wBACV;oBACF;iBACD;YAEHhB,eAAe;gBACb;oBACE,MAAM;oBACN,MAAM;gBACR;gBACA;oBACE,MAAM;oBACN,MAAM;gBACR;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKN;wBACL,QAAQ;oBACV;gBACF;mBACGqB;aACJ;QACH,OACEf,eAAe;YACb;gBACE,MAAM;gBACN,MAAM;YACR;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKN;oBACL,QAAQ;gBACV;YACF;SACD;QAGH,MAAMwB,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASlC;YAAa;YACxC;gBACE,MAAM;gBACN,SAASgB;YACX;SACD;QAED,IAAImB;QAGJ,IAAI;YACFA,MAAM,MAAMC,yBACVF,MACAnF;QAEJ,EAAE,OAAO0C,OAAO;YACd,MAAM4C,oBAAoBC,sCAAsC7C;YAChE,IAAI,CAAC4C,mBACH,MAAM5C;YAER/C,MAAM;YACN,OAAO2F;QACT;QAEA,MAAM,EAAEE,OAAO,EAAE,GAAGJ;QACpBhF,OAAO,CAACoF,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DpF,OAAOoF,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IApfA,YACEjF,OAA2D,EAC3DR,GAAoB,CACpB;QAPF;QAEA;QAMEK,OAAOG,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAM8D,QAAQ,OAAO,CAAC9D;QAGlD,IAAI,AAAyB,WAAlBR,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAueF"}
+{"version":3,"file":"service/index.mjs","sources":["../../../src/service/index.ts"],"sourcesContent":["import { defaultModelFamilyRequiredForLocateMessage } from '@/ai-model/errors';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  AiLocateSection,\n  buildSearchAreaConfig,\n} from '@/ai-model/inspect';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport {\n  AIResponseParseError,\n  callAIWithObjectResponse,\n} from '@/ai-model/service-caller';\nimport type { AIArgs } from '@/ai-model/types';\nimport type { SearchAreaConfig } from '@/ai-model/workflows/inspect/types';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  PlanningLocateParam,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport {\n  compositeElementInfoImg,\n  compositePointMarkerImg,\n  cropByRect,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionContentPart } from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  createServiceDump,\n  getDescribeDeepContextAreas,\n  getDescribeDeepLocateResizeSize,\n  getDescribeMarkerBorderThickness,\n  getDescribeMarkerRect,\n  getRectInCrop,\n  recoverDescribeResponseFromParseError,\n} from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\ninterface LocateSearchAreaResult {\n  config?: SearchAreaConfig;\n  trace: {\n    sourceRect?: Rect;\n    rawResponse?: string;\n    rawChoiceMessage?: unknown;\n    usage?: AIUsageInfo;\n  };\n}\n\nconst debug = getDebug('ai:service');\n\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: PlanningLocateParam,\n    opt: LocateOpts,\n    modelRuntime: ModelRuntime,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const { config: modelConfig } = modelRuntime;\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    if (!modelConfig.modelFamily) {\n      throw new Error(defaultModelFamilyRequiredForLocateMessage);\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    const searchArea = await this.resolveLocateSearchArea({\n      query,\n      queryPrompt,\n      opt,\n      context,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content,\n    } = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchArea.config,\n      modelRuntime,\n      abortSignal,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawChoiceMessage,\n      formatResponse: parseResult,\n      usage,\n      searchArea: searchArea.trace.sourceRect,\n      searchAreaRawResponse: searchArea.trace.rawResponse,\n      searchAreaRawChoiceMessage: searchArea.trace.rawChoiceMessage,\n      searchAreaUsage: searchArea.trace.usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea.trace.sourceRect,\n      error: errorLog,\n    };\n\n    const element = parseResult.element;\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: element ? [element] : [],\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (element) {\n      return {\n        element: {\n          center: element.center,\n          rect: element.rect,\n          description: element.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  private async resolveLocateSearchArea(options: {\n    query: PlanningLocateParam;\n    queryPrompt: TUserPrompt;\n    opt: LocateOpts;\n    context: UIContext;\n    modelRuntime: ModelRuntime;\n    abortSignal?: AbortSignal;\n  }): Promise<LocateSearchAreaResult> {\n    const { query, queryPrompt, opt, context, modelRuntime, abortSignal } =\n      options;\n    const { adapter } = modelRuntime;\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n    if (!query.deepLocate) {\n      return { trace: {} };\n    }\n\n    if (hasPlanLocatedElement) {\n      const config = await buildSearchAreaConfig({\n        context,\n        baseRect: opt.planLocatedElement!.rect,\n      });\n\n      return {\n        config,\n        trace: {\n          sourceRect: config.sourceRect,\n          rawResponse: JSON.stringify({\n            source: 'plan-located-element',\n            rect: opt.planLocatedElement!.rect,\n          }),\n        },\n      };\n    }\n\n    if (adapter.locate.supportsSearchArea) {\n      const searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: queryPrompt,\n        modelRuntime,\n        abortSignal,\n      });\n      const { searchAreaConfig } = searchAreaResponse;\n      assert(\n        searchAreaConfig,\n        `cannot find search area for \"${queryPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n\n      return {\n        config: searchAreaConfig,\n        trace: {\n          sourceRect: searchAreaConfig.sourceRect,\n          rawResponse: searchAreaResponse.rawResponse,\n          rawChoiceMessage: searchAreaResponse.rawChoiceMessage,\n          usage: searchAreaResponse.usage,\n        },\n      };\n    }\n\n    const firstPassLocateResult = await AiLocateElement({\n      context,\n      targetElementDescription: queryPrompt,\n      modelRuntime,\n      abortSignal,\n    });\n    assert(\n      firstPassLocateResult.rect,\n      `cannot find search area for \"${queryPrompt}\"${\n        firstPassLocateResult.parseResult.errors?.length\n          ? `: ${firstPassLocateResult.parseResult.errors.join('\\n')}`\n          : ''\n      }`,\n    );\n\n    const config = await buildSearchAreaConfig({\n      context,\n      baseRect: firstPassLocateResult.rect,\n    });\n\n    return {\n      config,\n      trace: {\n        sourceRect: config.sourceRect,\n        rawResponse: JSON.stringify({\n          source: 'deep-locate-first-pass',\n          rect: firstPassLocateResult.rect,\n          rawResponse: firstPassLocateResult.rawResponse,\n        }),\n        rawChoiceMessage: firstPassLocateResult.rawChoiceMessage,\n        usage: firstPassLocateResult.usage,\n      },\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let rawChoiceMessage: unknown;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelRuntime,\n        pageDescription,\n        abortSignal: executionOptions?.abortSignal,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      rawChoiceMessage = result.rawChoiceMessage;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          rawChoiceMessage: error.rawChoiceMessage,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      rawChoiceMessage,\n      formatResponse: parseResult,\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelRuntime: ModelRuntime,\n    opt?: {\n      deepDescribe?: boolean;\n      context?: UIContext;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = opt?.context || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetFromPoint = Array.isArray(target);\n    const targetRect: Rect = targetFromPoint\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n    const targetPoint = targetFromPoint\n      ? {\n          x: target[0],\n          y: target[1],\n        }\n      : undefined;\n\n    const usePointMarker = targetFromPoint;\n    const imagePayload = usePointMarker\n      ? await compositePointMarkerImg({\n          inputImgBase64: screenshotBase64,\n          size: shotSize,\n          point: targetPoint!,\n        })\n      : await compositeElementInfoImg({\n          inputImgBase64: screenshotBase64,\n          size: shotSize,\n          elementsPositionInfo: [\n            {\n              rect: getDescribeMarkerRect(targetRect),\n            },\n          ],\n          borderThickness: getDescribeMarkerBorderThickness(targetRect),\n          centerPoint: true,\n        });\n\n    const shouldDeepDescribe = opt?.deepDescribe;\n    let imageContent: ChatCompletionContentPart[];\n    if (shouldDeepDescribe) {\n      const contextAreas = getDescribeDeepContextAreas(targetRect, shotSize);\n      const contextImages = await Promise.all(\n        contextAreas.map(async (area) => {\n          debug('describe: cropping deep context area', area);\n          const croppedResult = await cropByRect(screenshotBase64, area.rect);\n          const cropSize = {\n            width: croppedResult.width,\n            height: croppedResult.height,\n          };\n          const targetInCrop = getRectInCrop(targetRect, area.rect, cropSize);\n          const markedCropPayload = targetFromPoint\n            ? await compositePointMarkerImg({\n                inputImgBase64: croppedResult.imageBase64,\n                size: cropSize,\n                point: {\n                  x: targetPoint!.x - area.rect.left,\n                  y: targetPoint!.y - area.rect.top,\n                },\n              })\n            : await compositeElementInfoImg({\n                inputImgBase64: croppedResult.imageBase64,\n                size: cropSize,\n                elementsPositionInfo: [\n                  {\n                    rect: getDescribeMarkerRect(targetInCrop),\n                  },\n                ],\n                borderThickness: getDescribeMarkerBorderThickness(targetInCrop),\n                centerPoint: true,\n              });\n          const resizeSize = getDescribeDeepLocateResizeSize(croppedResult);\n          return {\n            kind: area.kind,\n            imageBase64: resizeSize\n              ? await resizeImgBase64(markedCropPayload, resizeSize)\n              : markedCropPayload,\n          };\n        }),\n      );\n      const contextImageContent =\n        contextImages.flatMap<ChatCompletionContentPart>((item, index) => [\n          {\n            type: 'text',\n            text: `Image ${index + 2}: focused detail crop around the target, for reading text, icon shape, and exact local boundaries.`,\n          },\n          {\n            type: 'image_url',\n            image_url: {\n              url: item.imageBase64,\n              detail: 'high',\n            },\n          },\n        ]);\n\n      imageContent = [\n        {\n          type: 'text' as const,\n          text: 'Use these images together to describe the real UI target marked by the temporary callout. Do not describe the marker itself.',\n        },\n        {\n          type: 'text' as const,\n          text: 'Image 1: full screenshot overview with the target marker, for page position and ownership context.',\n        },\n        {\n          type: 'image_url' as const,\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        ...contextImageContent,\n      ];\n    } else {\n      imageContent = [\n        {\n          type: 'text' as const,\n          text: 'Full screenshot with a temporary callout marking the target:',\n        },\n        {\n          type: 'image_url' as const,\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ];\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: imageContent,\n      },\n    ];\n\n    let res: Awaited<\n      ReturnType<typeof callAIWithObjectResponse<AIDescribeElementResponse>>\n    >;\n    try {\n      res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n        msgs,\n        modelRuntime,\n      );\n    } catch (error) {\n      const recoveredResponse = recoverDescribeResponseFromParseError(error);\n      if (!recoveredResponse) {\n        throw error;\n      }\n      debug('describe: recovered malformed description JSON response');\n      return recoveredResponse;\n    }\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Service","query","opt","modelRuntime","abortSignal","modelConfig","queryPrompt","assert","Error","defaultModelFamilyRequiredForLocateMessage","context","searchArea","startTime","Date","parseResult","rect","rawResponse","rawChoiceMessage","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","errorLog","dumpData","element","dump","createServiceDump","ServiceError","options","adapter","hasPlanLocatedElement","config","buildSearchAreaConfig","JSON","searchAreaResponse","AiLocateSection","searchAreaConfig","firstPassLocateResult","dataDemand","pageDescription","multimodalPrompt","executionOptions","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetFromPoint","Array","targetRect","Math","targetPoint","undefined","usePointMarker","imagePayload","compositePointMarkerImg","compositeElementInfoImg","getDescribeMarkerRect","getDescribeMarkerBorderThickness","shouldDeepDescribe","imageContent","contextAreas","getDescribeDeepContextAreas","contextImages","Promise","area","croppedResult","cropByRect","cropSize","targetInCrop","getRectInCrop","markedCropPayload","resizeSize","getDescribeDeepLocateResizeSize","resizeImgBase64","contextImageContent","item","index","msgs","res","callAIWithObjectResponse","recoveredResponse","recoverDescribeResponseFromParseError","content"],"mappings":";;;;;;;;;;;;;;;;;;;AA0EA,MAAMA,QAAQC,SAAS;AAER,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,YAA0B,EAC1BC,WAAyB,EACM;QAC/B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGF;QAChC,MAAMG,cAAc,AAAiB,YAAjB,OAAOL,QAAqBA,QAAQA,MAAM,MAAM;QACpEM,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAON,OAAoB;QAElC,IAAI,CAACI,YAAY,WAAW,EAC1B,MAAM,IAAIG,MAAMC;QAGlB,MAAMC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,MAAMS,aAAa,MAAM,IAAI,CAAC,uBAAuB,CAAC;YACpDV;YACAK;YACAJ;YACAQ;YACAP;YACAC;QACF;QAEA,MAAMQ,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,gBAAgB;YACxBV;YACA,0BAA0BJ;YAC1B,cAAcK,WAAW,MAAM;YAC/BR;YACAC;QACF;QAEA,MAAMiB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZL;YACAC;YACA,gBAAgBH;YAChBI;YACA,YAAYP,WAAW,KAAK,CAAC,UAAU;YACvC,uBAAuBA,WAAW,KAAK,CAAC,WAAW;YACnD,4BAA4BA,WAAW,KAAK,CAAC,gBAAgB;YAC7D,iBAAiBA,WAAW,KAAK,CAAC,KAAK;YACvCQ;QACF;QAEA,IAAII;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,4BAA4B,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASlB;YACX;YACA,aAAaS;YACb,MAAM;YACNO;YACA,YAAY,CAAC,CAACX,WAAW,KAAK,CAAC,UAAU;YACzC,OAAOY;QACT;QAEA,MAAME,UAAUX,YAAY,OAAO;QAEnC,MAAMY,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC,UAAU;gBAACA;aAAQ,GAAG,EAAE;QAC1C;QAEA,IAAIF,UACF,MAAM,IAAIK,aAAaL,UAAUG;QAGnC,IAAID,SACF,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,MAAM;gBACtB,MAAMA,QAAQ,IAAI;gBAClB,aAAaA,QAAQ,WAAW;YAClC;YACAV;YACAW;QACF;QAGF,OAAO;YACL,SAAS;YACTX;YACAW;QACF;IACF;IAEA,MAAc,wBAAwBG,OAOrC,EAAmC;QAClC,MAAM,EAAE5B,KAAK,EAAEK,WAAW,EAAEJ,GAAG,EAAEQ,OAAO,EAAEP,YAAY,EAAEC,WAAW,EAAE,GACnEyB;QACF,MAAM,EAAEC,OAAO,EAAE,GAAG3B;QACpB,MAAM4B,wBAAwB,CAAC,CAAC7B,KAAK,oBAAoB;QAEzD,IAAI,CAACD,MAAM,UAAU,EACnB,OAAO;YAAE,OAAO,CAAC;QAAE;QAGrB,IAAI8B,uBAAuB;YACzB,MAAMC,SAAS,MAAMC,sBAAsB;gBACzCvB;gBACA,UAAUR,IAAI,kBAAkB,CAAE,IAAI;YACxC;YAEA,OAAO;gBACL8B;gBACA,OAAO;oBACL,YAAYA,OAAO,UAAU;oBAC7B,aAAaE,KAAK,SAAS,CAAC;wBAC1B,QAAQ;wBACR,MAAMhC,IAAI,kBAAkB,CAAE,IAAI;oBACpC;gBACF;YACF;QACF;QAEA,IAAI4B,QAAQ,MAAM,CAAC,kBAAkB,EAAE;YACrC,MAAMK,qBAAqB,MAAMC,gBAAgB;gBAC/C1B;gBACA,oBAAoBJ;gBACpBH;gBACAC;YACF;YACA,MAAM,EAAEiC,gBAAgB,EAAE,GAAGF;YAC7B5B,OACE8B,kBACA,CAAC,6BAA6B,EAAE/B,YAAY,CAAC,EAC3C6B,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAGJ,OAAO;gBACL,QAAQE;gBACR,OAAO;oBACL,YAAYA,iBAAiB,UAAU;oBACvC,aAAaF,mBAAmB,WAAW;oBAC3C,kBAAkBA,mBAAmB,gBAAgB;oBACrD,OAAOA,mBAAmB,KAAK;gBACjC;YACF;QACF;QAEA,MAAMG,wBAAwB,MAAMlB,gBAAgB;YAClDV;YACA,0BAA0BJ;YAC1BH;YACAC;QACF;QACAG,OACE+B,sBAAsB,IAAI,EAC1B,CAAC,6BAA6B,EAAEhC,YAAY,CAAC,EAC3CgC,sBAAsB,WAAW,CAAC,MAAM,EAAE,SACtC,CAAC,EAAE,EAAEA,sBAAsB,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,GAC1D,IACJ;QAGJ,MAAMN,SAAS,MAAMC,sBAAsB;YACzCvB;YACA,UAAU4B,sBAAsB,IAAI;QACtC;QAEA,OAAO;YACLN;YACA,OAAO;gBACL,YAAYA,OAAO,UAAU;gBAC7B,aAAaE,KAAK,SAAS,CAAC;oBAC1B,QAAQ;oBACR,MAAMI,sBAAsB,IAAI;oBAChC,aAAaA,sBAAsB,WAAW;gBAChD;gBACA,kBAAkBA,sBAAsB,gBAAgB;gBACxD,OAAOA,sBAAsB,KAAK;YACpC;QACF;IACF;IAEA,MAAM,QACJC,UAA+B,EAC/BpC,YAA0B,EAC1BD,GAA0B,EAC1BsC,eAAwB,EACxBC,gBAAoC,EACpC/B,OAAmB,EACnBgC,gBAEC,EACiC;QAClCnC,OAAOG,SAAS;QAChBH,OACE,AAAsB,YAAtB,OAAOgC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAM3B,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMwB,SAAS,MAAMC,qBAAwB;gBAC3ClC;gBACA,WAAW6B;gBACXE;gBACA,eAAevC;gBACfC;gBACAqC;gBACA,aAAaE,kBAAkB;YACjC;YACA5B,cAAc6B,OAAO,WAAW;YAChC3B,cAAc2B,OAAO,WAAW;YAChC1B,mBAAmB0B,OAAO,gBAAgB;YAC1CzB,QAAQyB,OAAO,KAAK;YACpBxB,oBAAoBwB,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,sBAAsB;gBAEzC,MAAMzB,WAAWR,KAAK,GAAG,KAAKD;gBAC9B,MAAMU,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAawB,MAAM,WAAW;oBAC9B,kBAAkBA,MAAM,gBAAgB;oBACxC,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMnB,OAAOC,kBAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEY;oBAAW;oBACxB,MAAM;oBACNjB;oBACA,OAAOuB,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIjB,aAAaiB,MAAM,OAAO,EAAEnB;YACxC;YACA,MAAMmB;QACR;QAEA,MAAMxB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZL;YACAC;YACA,gBAAgBH;YAChBI;YACAC;QACF;QAEA,IAAII;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,qBAAqB,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTe;YACF;YACA,MAAM;YACNjB;YACA,OAAOC;QACT;QAEA,MAAM,EAAEwB,IAAI,EAAEC,OAAO,EAAE,GAAGlC,eAAe,CAAC;QAE1C,MAAMY,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACXuB;QACF;QAEA,IAAIxB,YAAY,CAACwB,MACf,MAAM,IAAInB,aAAaL,UAAUG;QAGnC,OAAO;YACLqB;YACAC;YACA9B;YACAC;YACAO;QACF;IACF;IAEA,MAAM,SACJuB,MAA+B,EAC/B9C,YAA0B,EAC1BD,GAGC,EACwD;QACzDK,OAAO0C,QAAQ;QACf,MAAMvC,UAAUR,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAC9D,MAAM,EAAEgD,QAAQ,EAAE,GAAGxC;QACrB,MAAMyC,mBAAmBzC,QAAQ,UAAU,CAAC,MAAM;QAClDH,OAAO4C,kBAAkB;QACzB,MAAMC,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,kBAAkBC,MAAM,OAAO,CAACP;QACtC,MAAMQ,aAAmBF,kBACrB;YACE,MAAMG,KAAK,KAAK,CAACT,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKI,KAAK,KAAK,CAACT,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QACJ,MAAMU,cAAcJ,kBAChB;YACE,GAAGN,MAAM,CAAC,EAAE;YACZ,GAAGA,MAAM,CAAC,EAAE;QACd,IACAW;QAEJ,MAAMC,iBAAiBN;QACvB,MAAMO,eAAeD,iBACjB,MAAME,wBAAwB;YAC5B,gBAAgBZ;YAChB,MAAMD;YACN,OAAOS;QACT,KACA,MAAMK,wBAAwB;YAC5B,gBAAgBb;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMe,sBAAsBR;gBAC9B;aACD;YACD,iBAAiBS,iCAAiCT;YAClD,aAAa;QACf;QAEJ,MAAMU,qBAAqBjE,KAAK;QAChC,IAAIkE;QACJ,IAAID,oBAAoB;YACtB,MAAME,eAAeC,4BAA4Bb,YAAYP;YAC7D,MAAMqB,gBAAgB,MAAMC,QAAQ,GAAG,CACrCH,aAAa,GAAG,CAAC,OAAOI;gBACtB3E,MAAM,wCAAwC2E;gBAC9C,MAAMC,gBAAgB,MAAMC,WAAWxB,kBAAkBsB,KAAK,IAAI;gBAClE,MAAMG,WAAW;oBACf,OAAOF,cAAc,KAAK;oBAC1B,QAAQA,cAAc,MAAM;gBAC9B;gBACA,MAAMG,eAAeC,cAAcrB,YAAYgB,KAAK,IAAI,EAAEG;gBAC1D,MAAMG,oBAAoBxB,kBACtB,MAAMQ,wBAAwB;oBAC5B,gBAAgBW,cAAc,WAAW;oBACzC,MAAME;oBACN,OAAO;wBACL,GAAGjB,YAAa,CAAC,GAAGc,KAAK,IAAI,CAAC,IAAI;wBAClC,GAAGd,YAAa,CAAC,GAAGc,KAAK,IAAI,CAAC,GAAG;oBACnC;gBACF,KACA,MAAMT,wBAAwB;oBAC5B,gBAAgBU,cAAc,WAAW;oBACzC,MAAME;oBACN,sBAAsB;wBACpB;4BACE,MAAMX,sBAAsBY;wBAC9B;qBACD;oBACD,iBAAiBX,iCAAiCW;oBAClD,aAAa;gBACf;gBACJ,MAAMG,aAAaC,gCAAgCP;gBACnD,OAAO;oBACL,MAAMD,KAAK,IAAI;oBACf,aAAaO,aACT,MAAME,gBAAgBH,mBAAmBC,cACzCD;gBACN;YACF;YAEF,MAAMI,sBACJZ,cAAc,OAAO,CAA4B,CAACa,MAAMC,QAAU;oBAChE;wBACE,MAAM;wBACN,MAAM,CAAC,MAAM,EAAEA,QAAQ,EAAE,kGAAkG,CAAC;oBAC9H;oBACA;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKD,KAAK,WAAW;4BACrB,QAAQ;wBACV;oBACF;iBACD;YAEHhB,eAAe;gBACb;oBACE,MAAM;oBACN,MAAM;gBACR;gBACA;oBACE,MAAM;oBACN,MAAM;gBACR;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKN;wBACL,QAAQ;oBACV;gBACF;mBACGqB;aACJ;QACH,OACEf,eAAe;YACb;gBACE,MAAM;gBACN,MAAM;YACR;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKN;oBACL,QAAQ;gBACV;YACF;SACD;QAGH,MAAMwB,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASlC;YAAa;YACxC;gBACE,MAAM;gBACN,SAASgB;YACX;SACD;QAED,IAAImB;QAGJ,IAAI;YACFA,MAAM,MAAMC,yBACVF,MACAnF;QAEJ,EAAE,OAAO0C,OAAO;YACd,MAAM4C,oBAAoBC,sCAAsC7C;YAChE,IAAI,CAAC4C,mBACH,MAAM5C;YAER/C,MAAM;YACN,OAAO2F;QACT;QAEA,MAAM,EAAEE,OAAO,EAAE,GAAGJ;QACpBhF,OAAO,CAACoF,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DpF,OAAOoF,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IApfA,YACEjF,OAA2D,EAC3DR,GAAoB,CACpB;QAPF;QAEA;QAMEK,OAAOG,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAM8D,QAAQ,OAAO,CAAC9D;QAGlD,IAAI,AAAyB,WAAlBR,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAueF"}

package/dist/es/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport type { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport { ServiceError } from './errors';\nexport {\n  ExecutionDump,\n  ReportActionDump,\n  GroupedActionDump,\n} from './dump/report-action-dump';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  /**\n   * Raw top-level `.model` value returned by the model service response.\n   */\n  response_model_name: string | undefined;\n  /**\n   * Semantic intent of the model call, such as default, planning, or insight.\n   */\n  intent: string | undefined;\n  /**\n   * Config slot where the model config was resolved from. For example, a\n   * planning call may use the default slot when no planning model is configured.\n   */\n  slot: string | undefined;\n  request_id: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type LocateResultPoint = [number, number];\nexport type Bbox = [number, number, number, number];\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\n\nexport interface AIElementLocateResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  errors?: string[];\n}\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  references_bbox?: LocateResultBbox[];\n  references_point?: LocateResultPoint[];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n  includedInRect?: boolean;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepLocate: boolean;\n  deepDescribe: boolean;\n  verifyResult?: LocateValidatorResult;\n  success: boolean;\n  error?: string;\n  failureStage?: 'describe' | 'verify';\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  /**\n   * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n   */\n  abstract screenshot: ScreenshotItem;\n\n  /**\n   * screenshot size after shrinking\n   */\n  abstract shotSize: Size;\n\n  /**\n   * The ratio for converting shrunk screenshot coordinates to logical coordinates.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - User-defined screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n   * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n   */\n  abstract shrunkShotToLogicalRatio: number;\n\n  abstract _isFrozen?: boolean;\n\n  // @deprecated - backward compatibility for aiLocate\n  abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  /**\n   * Adapter-extracted content from the search-area model call. This is not the\n   * full provider response or choices[0].message.\n   */\n  searchAreaRawResponse?: string;\n  searchAreaRawChoiceMessage?: unknown;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport type ReportAttributes = Record<\n  string,\n  string | number | boolean | null | undefined\n>;\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: ReportAttributes;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement?: LocateResultElement[];\n  matchedRect?: Rect;\n  deepLocate?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n  context?: string;\n  abortSignal?: AbortSignal;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n}\n\nexport type PlanningLocateParamWithLocatedPixelBbox = PlanningLocateParam & {\n  /** Pixel bbox of the located element in screenshot coordinates. */\n  locatedPixelBbox: PixelBbox;\n};\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n  param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' | 'running' | 'finished';\n\nexport interface SubGoal {\n  index: number;\n  status: SubGoalStatus;\n  description: string;\n  logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  thought?: string;\n  log: string;\n  memory?: string;\n  error?: string;\n  finalizeMessage?: string;\n  finalizeSuccess?: boolean;\n  updateSubGoals?: SubGoal[];\n  markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: string;\n  rawChoiceMessage?: unknown;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n  output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\n/**\n * Generic agent progress bus.\n *\n * Progress notifications are a single, generic stream the agent broadcasts as\n * it works. Each event is a thin envelope: a `scope` naming the producer, a\n * `phase` within that producer's lifecycle, a monotonic `sequence`, and a\n * structured, presentation-free `data` payload. The bus knows nothing about any\n * particular producer's payload - consumers narrow by `scope`. `aiAct` is the\n * first producer (\"pilot\") on this bus; others (queries, waits, ...) can be\n * added without touching the bus, the listener API, or the renderer core.\n */\nexport interface AgentProgressEvent<\n  TScope extends string = string,\n  TData = unknown,\n  TPhase extends string = string,\n> {\n  scope: TScope;\n  phase: TPhase;\n  sequence: number;\n  data: TData;\n}\n\nexport type AgentProgressListener<\n  TScope extends string = string,\n  TData = unknown,\n  TPhase extends string = string,\n> = (event: AgentProgressEvent<TScope, TData, TPhase>) => Promise<void> | void;\n\n// --- aiAct: the first producer on the generic progress bus ---\n\nexport type AiActProgressPhase =\n  | 'start'\n  | 'plan_thinking'\n  | 'plan_planned'\n  | 'plan_action'\n  | 'plan_failed'\n  | 'action_running'\n  | 'action_done'\n  | 'action_failed'\n  | 'complete'\n  | 'failed';\n\nexport interface AiActProgressAction {\n  name: string;\n  target?: string;\n  point?: [number, number];\n  bbox?: [number, number, number, number];\n  /**\n   * Structured, compacted summary of the action params for actions that are\n   * not described by a point/bbox (e.g. `Sleep` -> `{ timeMs }`). This is data,\n   * never a pre-formatted display string; consumers decide how to render it.\n   */\n  param?: unknown;\n}\n\n/**\n * Structured payload carried by aiAct progress events. The producer only\n * reports *what happened* as data: the action involved, the raw text the model\n * produced, timings and errors. It never assembles human-readable log lines or\n * truncates strings for display - that belongs to whichever layer consumes the\n * stream (e.g. the CLI verbose renderer).\n */\nexport interface AiActProgressData {\n  /** Original user instruction, present on the `start` phase. */\n  prompt?: string;\n  planIndex?: number;\n  planLimit?: number;\n  /** Latest screenshot, present on `plan_thinking`. */\n  screenshot?: ScreenshotItem;\n  /** Structured action descriptor, present on the `*action*` phases. */\n  action?: AiActProgressAction;\n  /**\n   * Raw, untruncated semantic text produced by the model. Consumers choose\n   * which field to surface and how to format/truncate it.\n   */\n  thought?: string;\n  log?: string;\n  output?: string;\n  /** Wall-clock cost of an action, present on `action_done`/`action_failed`. */\n  durationMs?: number;\n  error?: string;\n}\n\nexport const aiActProgressScope = 'aiAct';\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  description?: string;\n  timing?: string;\n}\n\nexport interface RecordToReportScreenshot {\n  /**\n   * PNG/JPEG data URI, or raw PNG base64 body.\n   */\n  base64: string;\n  description?: string;\n}\n\nexport interface RecordToReportOptions {\n  content?: string;\n  /**\n   * @deprecated Use `screenshots: [{ base64 }]` instead.\n   */\n  screenshotBase64?: string;\n  /**\n   * Custom screenshots to display under a single report entry.\n   */\n  screenshots?: RecordToReportScreenshot[];\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    taskId: string;\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    /**\n     * Optional feedback produced by a task for the next planning round.\n     * This is execution metadata, not part of the action return value.\n     */\n    planningFeedback?: string;\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      getUiContextStart?: number;\n      getUiContextEnd?: number;\n      callAiStart?: number;\n      callAiEnd?: number;\n      beforeInvokeActionHookStart?: number;\n      beforeInvokeActionHookEnd?: number;\n      callActionStart?: number;\n      callActionEnd?: number;\n      afterInvokeActionHookStart?: number;\n      afterInvokeActionHookEnd?: number;\n      captureAfterCallingSnapshotStart?: number;\n      captureAfterCallingSnapshotEnd?: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    /**\n     * Pixel rect of the deepLocate first-stage search area in screenshot\n     * coordinates. Used by reports to explain the crop/zoom area that the\n     * final locate ran against.\n     */\n    searchArea?: Rect;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  /** Stable unique identifier for this execution run */\n  id?: string;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n  domIncluded?: boolean | 'visible-only';\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport interface ExecutionTaskPlanningParam {\n  userInstruction: TUserPrompt;\n  userInstructionDisplay?: string;\n  replanningCycleLimit?: number;\n  aiActContext?: string;\n  imagesIncludeCount?: number;\n  deepThink?: DeepThinkOption;\n  subGoalStatus?: string;\n  memoriesStatus?: string;\n}\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningParam,\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nHow a report file stores screenshots:\n- `inline`: base64 image script tags embedded in the single HTML file\n- `directory`: external PNG files under a sibling `screenshots/` dir\n*/\nexport type ScreenshotMode = 'inline' | 'directory';\n\n/*\nReport metadata - extracted from ReportActionDump for per-execution writes\n*/\nexport interface ReportMeta {\n  groupName: string;\n  groupDescription?: string;\n  sdkVersion: string;\n  modelBriefs: ModelBrief[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupMeta = ReportMeta;\n\n/*\nReport dump\n*/\nexport interface IReportActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: ModelBrief[];\n  executions: IExecutionDump[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type IGroupedActionDump = IReportActionDump;\n\nexport interface ModelBrief {\n  /**\n   * The intent/category of the model call, for example \"planning\" or \"insight\".\n   */\n  intent?: string;\n\n  /**\n   * The model name returned by usage metadata, for example \"gpt-4o\".\n   */\n  name?: string;\n\n  /**\n   * Optional human-readable model description, for example \"qwen2.5-vl mode\".\n   */\n  modelDescription?: string;\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (\n    param: TParam,\n    context?: ExecutorContext,\n  ) => Promise<TReturn> | TReturn;\n  delayBeforeRunner?: number;\n  delayAfterRunner?: number;\n  /**\n   * An example param object for this action.\n   * Locate fields with { prompt } may be resolved to internal pixel bboxes when needed.\n   */\n  sample?: { [K in keyof TParam]?: any };\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n  /**\n   * Optional cache directory path.\n   * When set, cache files are written to this directory instead of\n   * `<MIDSCENE_RUN_DIR>/cache`.\n   */\n  cacheDir?: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  // @deprecated Use `reportFileName` and `cache.id` instead.\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if persist per-execution dump files next to the report, default false */\n  persistExecutionDump?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n\n  /**\n   * Use directory-based report format with separate image files.\n   *\n   * When enabled:\n   * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n   * - Report is generated as `index.html` with relative image paths\n   * - Reduces memory usage and report file size\n   *\n   * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n   * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n   * work due to browser CORS restrictions.\n   *\n   * @default 'single-html'\n   */\n  outputFormat?: 'single-html' | 'html-and-external-assets';\n\n  onTaskStartTip?: OnTaskStartTip;\n  aiActContext?: string;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  reportAttributes?: ReportAttributes;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults are resolved by the active model adapter: 20 for standard planning,\n   * 40 for UI-TARS, and 100 for Auto-GLM.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Wait time in milliseconds after each action execution.\n   * This allows the UI to settle and stabilize before the next action.\n   * Defaults to 300ms when not provided.\n   */\n  waitAfterAction?: number;\n\n  /**\n   * When set to true, Midscene will use the target device's formatted local\n   * time instead of the runtime system time. The target interface must implement\n   * getDeviceLocalTimeString to provide device-local wall-clock time.\n   * Default: false\n   */\n  useDeviceTime?: boolean;\n\n  /**\n   * Custom screenshot shrink factor to reduce AI token usage.\n   * When set, the screenshot will be scaled down by this factor from the physical resolution.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - AI analyzes the 1500px screenshot\n   * - Coordinates are transformed back to logical (500px) before actions execute\n   *\n   * Benefits:\n   * - Reduces token usage for high-resolution screenshots\n   * - Maintains accuracy by scaling coordinates appropriately\n   *\n   * Must be >= 1 (shrinking only, enlarging is not supported).\n   *\n   * @default 1 (no shrinking, uses original physical screenshot)\n   */\n  screenshotShrinkFactor?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileAttributes {\n  testDuration: number;\n  testStatus: TestStatus;\n  testTitle: string;\n  testId: string;\n  testDescription: string;\n}\n\nexport type ReportFileWithAttributes =\n  | {\n      reportFilePath: string;\n      reportAttributes: ReportFileAttributes;\n    }\n  | {\n      reportFilePath?: string;\n      reportAttributes: ReportFileAttributes & { testStatus: 'skipped' };\n    };\n"],"names":["UIContext","aiActProgressScope"],"mappings":";;;AAoIO,MAAeA;AA4BtB;AAsTO,MAAMC,qBAAqB"}
+{"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport type { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport { ServiceError } from './errors';\nexport {\n  ExecutionDump,\n  ReportActionDump,\n  GroupedActionDump,\n} from './dump/report-action-dump';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  /**\n   * Raw top-level `.model` value returned by the model service response.\n   */\n  response_model_name: string | undefined;\n  /**\n   * Semantic intent of the model call, such as default, planning, or insight.\n   */\n  intent: string | undefined;\n  /**\n   * Config slot where the model config was resolved from. For example, a\n   * planning call may use the default slot when no planning model is configured.\n   */\n  slot: string | undefined;\n  request_id: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type LocateResultPoint = [number, number];\nexport type Bbox = [number, number, number, number];\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\n\nexport interface AIElementLocateResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  errors?: string[];\n}\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n  references_bbox?: LocateResultBbox[];\n  references_point?: LocateResultPoint[];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n  includedInRect?: boolean;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepLocate: boolean;\n  deepDescribe: boolean;\n  verifyResult?: LocateValidatorResult;\n  success: boolean;\n  error?: string;\n  failureStage?: 'describe' | 'verify';\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  /**\n   * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n   */\n  abstract screenshot: ScreenshotItem;\n\n  /**\n   * screenshot size after shrinking\n   */\n  abstract shotSize: Size;\n\n  /**\n   * The ratio for converting shrunk screenshot coordinates to logical coordinates.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - User-defined screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n   * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n   */\n  abstract shrunkShotToLogicalRatio: number;\n\n  abstract _isFrozen?: boolean;\n\n  // @deprecated - backward compatibility for aiLocate\n  abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: unknown;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: unknown;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  /**\n   * Adapter-extracted content from the search-area model call. This is not the\n   * full provider response or choices[0].message.\n   */\n  searchAreaRawResponse?: string;\n  searchAreaRawChoiceMessage?: unknown;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport type ReportAttributes = Record<\n  string,\n  string | number | boolean | null | undefined\n>;\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: ReportAttributes;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement?: LocateResultElement[];\n  matchedRect?: Rect;\n  deepLocate?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n  context?: string;\n  abortSignal?: AbortSignal;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: LocateResultBbox;\n  point?: LocateResultPoint;\n}\n\nexport type PlanningLocateParamWithLocatedPixelBbox = PlanningLocateParam & {\n  /** Pixel bbox of the located element in screenshot coordinates. */\n  locatedPixelBbox: PixelBbox;\n};\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n  param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' | 'running' | 'finished';\n\nexport interface SubGoal {\n  index: number;\n  status: SubGoalStatus;\n  description: string;\n  logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  thought?: string;\n  log: string;\n  memory?: string;\n  error?: string;\n  finalizeMessage?: string;\n  finalizeSuccess?: boolean;\n  updateSubGoals?: SubGoal[];\n  markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse?: string;\n  rawChoiceMessage?: unknown;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n  output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\n/**\n * Generic agent progress bus.\n *\n * Progress notifications are a single, generic stream the agent broadcasts as\n * it works. Each event is a thin envelope: a `scope` naming the producer, a\n * `phase` within that producer's lifecycle, a monotonic `sequence`, and a\n * structured, presentation-free `data` payload. The bus knows nothing about any\n * particular producer's payload - consumers narrow by `scope`. `aiAct` is the\n * first producer (\"pilot\") on this bus; others (queries, waits, ...) can be\n * added without touching the bus, the listener API, or the renderer core.\n */\nexport interface AgentProgressEvent<\n  TScope extends string = string,\n  TData = unknown,\n  TPhase extends string = string,\n> {\n  scope: TScope;\n  phase: TPhase;\n  sequence: number;\n  data: TData;\n}\n\nexport type AgentProgressListener<\n  TScope extends string = string,\n  TData = unknown,\n  TPhase extends string = string,\n> = (event: AgentProgressEvent<TScope, TData, TPhase>) => Promise<void> | void;\n\n// --- aiAct: the first producer on the generic progress bus ---\n\nexport type AiActProgressPhase =\n  | 'start'\n  | 'plan_thinking'\n  | 'plan_planned'\n  | 'plan_action'\n  | 'plan_failed'\n  | 'action_running'\n  | 'action_done'\n  | 'action_failed'\n  | 'complete'\n  | 'failed';\n\nexport interface AiActProgressAction {\n  name: string;\n  target?: string;\n  point?: [number, number];\n  bbox?: [number, number, number, number];\n  /**\n   * Structured, compacted summary of the action params for actions that are\n   * not described by a point/bbox (e.g. `Sleep` -> `{ timeMs }`). This is data,\n   * never a pre-formatted display string; consumers decide how to render it.\n   */\n  param?: unknown;\n}\n\n/**\n * Structured payload carried by aiAct progress events. The producer only\n * reports *what happened* as data: the action involved, the raw text the model\n * produced, timings and errors. It never assembles human-readable log lines or\n * truncates strings for display - that belongs to whichever layer consumes the\n * stream (e.g. the CLI verbose renderer).\n */\nexport interface AiActProgressData {\n  /** Original user instruction, present on the `start` phase. */\n  prompt?: string;\n  planIndex?: number;\n  planLimit?: number;\n  /** Latest screenshot, present on `plan_thinking`. */\n  screenshot?: ScreenshotItem;\n  /** Structured action descriptor, present on the `*action*` phases. */\n  action?: AiActProgressAction;\n  /**\n   * Raw, untruncated semantic text produced by the model. Consumers choose\n   * which field to surface and how to format/truncate it.\n   */\n  thought?: string;\n  log?: string;\n  output?: string;\n  /** Wall-clock cost of an action, present on `action_done`/`action_failed`. */\n  durationMs?: number;\n  error?: string;\n}\n\nexport const aiActProgressScope = 'aiAct';\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  description?: string;\n  timing?: string;\n}\n\nexport interface RecordToReportScreenshot {\n  /**\n   * PNG/JPEG data URI, or raw PNG base64 body.\n   */\n  base64: string;\n  description?: string;\n}\n\nexport interface RecordToReportOptions {\n  content?: string;\n  /**\n   * @deprecated Use `screenshots: [{ base64 }]` instead.\n   */\n  screenshotBase64?: string;\n  /**\n   * Custom screenshots to display under a single report entry.\n   */\n  screenshots?: RecordToReportScreenshot[];\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    taskId: string;\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    /**\n     * Optional feedback produced by a task for the next planning round.\n     * This is execution metadata, not part of the action return value.\n     */\n    planningFeedback?: string;\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      getUiContextStart?: number;\n      getUiContextEnd?: number;\n      callAiStart?: number;\n      callAiEnd?: number;\n      beforeInvokeActionHookStart?: number;\n      beforeInvokeActionHookEnd?: number;\n      callActionStart?: number;\n      callActionEnd?: number;\n      afterInvokeActionHookStart?: number;\n      afterInvokeActionHookEnd?: number;\n      captureAfterCallingSnapshotStart?: number;\n      captureAfterCallingSnapshotEnd?: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    /**\n     * Pixel rect of the deepLocate first-stage search area in screenshot\n     * coordinates. Used by reports to explain the crop/zoom area that the\n     * final locate ran against.\n     */\n    searchArea?: Rect;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  /** Stable unique identifier for this execution run */\n  id?: string;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n  domIncluded?: boolean | 'visible-only';\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport interface ExecutionTaskPlanningParam {\n  userInstruction: TUserPrompt;\n  userInstructionDisplay?: string;\n  replanningCycleLimit?: number;\n  aiActContext?: string;\n  imagesIncludeCount?: number;\n  deepThink?: DeepThinkOption;\n  subGoalStatus?: string;\n  memoriesStatus?: string;\n}\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningParam,\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nHow a report file stores screenshots:\n- `inline`: base64 image script tags embedded in the single HTML file\n- `directory`: external PNG files under a sibling `screenshots/` dir\n*/\nexport type ScreenshotMode = 'inline' | 'directory';\n\n/*\nReport metadata - extracted from ReportActionDump for per-execution writes\n*/\nexport interface ReportMeta {\n  groupName: string;\n  groupDescription?: string;\n  sdkVersion: string;\n  modelBriefs: ModelBrief[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupMeta = ReportMeta;\n\n/*\nReport dump\n*/\nexport interface IReportActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: ModelBrief[];\n  executions: IExecutionDump[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type IGroupedActionDump = IReportActionDump;\n\nexport interface ModelBrief {\n  /**\n   * The intent/category of the model call, for example \"planning\" or \"insight\".\n   */\n  intent?: string;\n\n  /**\n   * The model name returned by usage metadata, for example \"gpt-4o\".\n   */\n  name?: string;\n\n  /**\n   * Optional human-readable model description, for example \"qwen2.5-vl mode\".\n   */\n  modelDescription?: string;\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (\n    param: TParam,\n    context?: ExecutorContext,\n  ) => Promise<TReturn> | TReturn;\n  delayBeforeRunner?: number;\n  delayAfterRunner?: number;\n  /**\n   * An example param object for this action.\n   * Locate fields with { prompt } may be resolved to internal pixel bboxes when needed.\n   */\n  sample?: { [K in keyof TParam]?: any };\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n  /**\n   * Optional cache directory path.\n   * When set, cache files are written to this directory instead of\n   * `<MIDSCENE_RUN_DIR>/cache`.\n   */\n  cacheDir?: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  // @deprecated Use `reportFileName` and `cache.id` instead.\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if persist per-execution dump files next to the report, default false */\n  persistExecutionDump?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n\n  /**\n   * Use directory-based report format with separate image files.\n   *\n   * When enabled:\n   * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n   * - Report is generated as `index.html` with relative image paths\n   * - Reduces memory usage and report file size\n   *\n   * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n   * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n   * work due to browser CORS restrictions.\n   *\n   * @default 'single-html'\n   */\n  outputFormat?: 'single-html' | 'html-and-external-assets';\n\n  onTaskStartTip?: OnTaskStartTip;\n  aiActContext?: string;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  reportAttributes?: ReportAttributes;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults are resolved by the active model adapter: 20 for standard planning,\n   * 40 for UI-TARS, and 100 for Auto-GLM.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Wait time in milliseconds after each action execution.\n   * This allows the UI to settle and stabilize before the next action.\n   * Defaults to 300ms when not provided.\n   */\n  waitAfterAction?: number;\n\n  /**\n   * When set to true, Midscene will use the target device's formatted local\n   * time instead of the runtime system time. The target interface must implement\n   * getDeviceLocalTimeString to provide device-local wall-clock time.\n   * Default: false\n   */\n  useDeviceTime?: boolean;\n\n  /**\n   * Custom screenshot shrink factor to reduce AI token usage.\n   * When set, the screenshot will be scaled down by this factor from the physical resolution.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - AI analyzes the 1500px screenshot\n   * - Coordinates are transformed back to logical (500px) before actions execute\n   *\n   * Benefits:\n   * - Reduces token usage for high-resolution screenshots\n   * - Maintains accuracy by scaling coordinates appropriately\n   *\n   * Must be >= 1 (shrinking only, enlarging is not supported).\n   *\n   * @default 1 (no shrinking, uses original physical screenshot)\n   */\n  screenshotShrinkFactor?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n\n  /**\n   * Called once per LLM call as soon as its usage is available, with the raw\n   * {@link AIUsageInfo} (token counts, model name, intent, request id, etc.).\n   *\n   * Use this for real-time, per-spec cost observability — e.g. push each call\n   * to Langfuse without waiting for the run to finish. For aggregated totals,\n   * read `agent.metrics` instead.\n   */\n  onLLMUsage?: (usage: AIUsageInfo) => void;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileAttributes {\n  testDuration: number;\n  testStatus: TestStatus;\n  testTitle: string;\n  testId: string;\n  testDescription: string;\n}\n\nexport type ReportFileWithAttributes =\n  | {\n      reportFilePath: string;\n      reportAttributes: ReportFileAttributes;\n    }\n  | {\n      reportFilePath?: string;\n      reportAttributes: ReportFileAttributes & { testStatus: 'skipped' };\n    };\n"],"names":["UIContext","aiActProgressScope"],"mappings":";;;AAoIO,MAAeA;AA4BtB;AAsTO,MAAMC,qBAAqB"}