@midscene/core 1.17.6 → 1.17.7

package/dist/es/agent/agent-report-manager.mjs

@@ -58,7 +58,7 @@ class AgentReportManager {
             report.session = session;
             report.file = void 0;
         }
-        report.generator.enableSourceSnapshots?.();
+        report.generator.enableSourceSnapshots?.(report.yamlScopeId);
     }
     async _createReportSource(scopeId, executionIds) {
         return this.reportState.generator.createSourceSnapshot?.(scopeId, executionIds);

package/dist/es/agent/agent-report-manager.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/agent-report-manager.mjs","sources":["../../../src/agent/agent-report-manager.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert, ifInBrowser, uuid } from '@midscene/shared/utils';\nimport { IS_REPORT_BUILD } from '../constants';\nimport { generateTestRunReportScriptTag } from '../dump/html-utils';\nimport {\n  type ReportSession,\n  currentReportSession,\n} from '../dump/report-session';\nimport {\n  type IReportGenerator,\n  ReportGenerator,\n  assertReportGenerationOptions,\n  nullReportGenerator,\n} from '../report-generator';\nimport type { TaskRunner } from '../task-runner';\nimport { WorkflowError } from '../test-runner/errors';\nimport type { WorkflowExecutionRecord } from '../test-runner/execution-record';\nimport { buildAgentTestRunReportDump } from '../test-runner/reporting/agent-report';\nimport { executionRecordsToReportInput } from '../test-runner/reporting/execution-record';\nimport {\n  type AgentOpt,\n  type ExecutionDump,\n  ReportActionDump,\n  type ReportMeta,\n} from '../types';\nimport { getVersion, reportHTMLContent } from '../utils';\nimport { getReportFileName } from './utils';\n\nconst warn = getDebug('agent', { console: true });\n\ntype ReportOptions = Pick<\n  AgentOpt,\n  | 'groupName'\n  | 'groupDescription'\n  | 'generateReport'\n  | 'persistExecutionDump'\n  | 'outputFormat'\n  | 'autoPrintReportMsg'\n  | 'reportAttributes'\n  | 'reportFileName'\n  | 'testId'\n>;\n\ninterface AgentReportContext {\n  getOptions(): ReportOptions;\n  getDeviceType(): ReportActionDump['deviceType'];\n  isDestroyed(): boolean;\n  isExternallyBusy(): boolean;\n}\n\ninterface AgentReportState {\n  dump: ReportActionDump;\n  file?: string | null;\n  fileName?: string;\n  generator: IReportGenerator;\n  session?: ReportSession;\n  finalization?: Promise<string | undefined>;\n  lastExecution?: ExecutionDump;\n  executionIndexByRunner: WeakMap<TaskRunner, number>;\n  yamlScopeId: string;\n  yamlRecords: WorkflowExecutionRecord[];\n  activeOperations: Set<Promise<unknown>>;\n}\n\n/** Owns report state and persistence; execution and device ownership stay with Agent. */\nexport class AgentReportManager {\n  private reportState: AgentReportState;\n  private reportReset?: Promise<ReportActionDump>;\n  private readonly reportByRunner = new WeakMap<TaskRunner, AgentReportState>();\n  private dumpUpdateListeners: Array<\n    (dump: string, executionDump?: ExecutionDump) => void\n  > = [];\n\n  constructor(private readonly context: AgentReportContext) {\n    const opts = this.opts;\n    const fileName =\n      opts.reportFileName ??\n      getReportFileName(opts.testId || context.getDeviceType() || 'web');\n    this.reportState = this.createReportState(\n      fileName,\n      opts.reportAttributes?.['data-group-id'] === fileName,\n    );\n  }\n\n  private get opts(): ReportOptions {\n    return this.context.getOptions();\n  }\n\n  get dump(): ReportActionDump {\n    return this.reportState.dump;\n  }\n  set dump(value: ReportActionDump) {\n    this.reportState.dump = value;\n  }\n\n  get reportFile(): string | null | undefined {\n    return this.reportState.file;\n  }\n  set reportFile(value: string | null | undefined) {\n    this.reportState.file = value;\n  }\n\n  get reportFileName(): string | undefined {\n    return this.reportState.fileName;\n  }\n  set reportFileName(value: string | undefined) {\n    this.reportState.fileName = value;\n  }\n\n  /** @internal Runner snapshots retain immutable data and shared image files. */\n  async _prepareForTestRunner(): Promise<void> {\n    const report = this.reportState;\n    const session = currentReportSession();\n    if (session && session !== report.session) {\n      if (report.session)\n        throw new Error(\n          'An Agent cannot be shared by different Test report sessions. Await resetDump() between runs.',\n        );\n      this.assertReportIdle();\n      await report.generator.finalize();\n      const recorder = session.createRecorder(() => this.isReportBusy(report));\n      // The previous standalone report owns its history. The Test stream starts\n      // at acquisition, so previous script operations do not enter this run.\n      this.clearDumpInMemory(report);\n      report.generator = recorder;\n      report.session = session;\n      report.file = undefined;\n    }\n    report.generator.enableSourceSnapshots?.();\n  }\n  async _createReportSource(scopeId: string, executionIds?: readonly string[]) {\n    return this.reportState.generator.createSourceSnapshot?.(\n      scopeId,\n      executionIds,\n    );\n  }\n\n  /** @internal The Agent owns one report stream across root YAML invocations. */\n  async _writeYamlExecutionReport(\n    record: WorkflowExecutionRecord,\n  ): Promise<string | undefined> {\n    const report = this.reportState;\n    const existingIndex = report.yamlRecords.findIndex(\n      (entry) => entry.runId === record.runId,\n    );\n    const recordIndex =\n      existingIndex < 0 ? report.yamlRecords.length : existingIndex;\n    report.yamlRecords[recordIndex] = record;\n    try {\n      const path = await report.generator.writeRunnerReport?.(\n        this.buildYamlReportInput(),\n        report.yamlScopeId,\n        this.getReportMeta(),\n      );\n      report.file = path ?? report.file;\n      return path;\n    } catch (error) {\n      report.yamlRecords[recordIndex] = Object.freeze({\n        ...record,\n        status: 'failed',\n        reportError: error,\n      });\n      throw error;\n    }\n  }\n\n  private buildYamlReportInput() {\n    const scopedRecords = this.reportState.yamlRecords.map((entry) => ({\n      ...entry,\n      ...(entry.execution\n        ? {\n            execution: {\n              document: {\n                ...entry.execution.document,\n                reportScopeId: this.reportState.yamlScopeId,\n              },\n              cases: entry.execution.cases.map((outcome) => ({\n                ...outcome,\n                ...(outcome.run\n                  ? {\n                      run: {\n                        ...outcome.run,\n                        reportScopeId: this.reportState.yamlScopeId,\n                      },\n                    }\n                  : {}),\n                ...(outcome.attempts\n                  ? {\n                      attempts: outcome.attempts.map((attempt) => ({\n                        ...attempt,\n                        reportScopeId: this.reportState.yamlScopeId,\n                      })),\n                    }\n                  : {}),\n              })),\n            },\n          }\n        : {}),\n    }));\n    return executionRecordsToReportInput(scopedRecords, {\n      runId: this.reportState.yamlScopeId,\n    });\n  }\n\n  private createEmptyDump(): ReportActionDump {\n    return new ReportActionDump({\n      sdkVersion: getVersion(),\n      groupName: this.opts.groupName!,\n      groupDescription: this.opts.groupDescription,\n      executions: [],\n      modelBriefs: [],\n      deviceType: this.context.getDeviceType(),\n    });\n  }\n\n  private createReportState(\n    fileName: string,\n    reuseExistingReport = false,\n  ): AgentReportState {\n    assertReportGenerationOptions(this.opts);\n    const session = currentReportSession();\n    const state: AgentReportState = {\n      dump: this.createEmptyDump(),\n      fileName,\n      generator: session\n        ? nullReportGenerator\n        : ReportGenerator.create(fileName, {\n            generateReport: this.opts.generateReport,\n            persistExecutionDump: this.opts.persistExecutionDump,\n            outputFormat: this.opts.outputFormat,\n            autoPrintReportMsg: this.opts.autoPrintReportMsg,\n            reuseExistingReport,\n          }),\n      executionIndexByRunner: new WeakMap(),\n      yamlScopeId: uuid(),\n      yamlRecords: [],\n      activeOperations: new Set(),\n    };\n    if (session) {\n      state.generator = session.createRecorder(() => this.isReportBusy(state));\n      state.session = session;\n    }\n    return state;\n  }\n\n  private isReportBusy(report: AgentReportState): boolean {\n    return report.activeOperations.size > 0 || this.context.isExternallyBusy();\n  }\n\n  private clearDumpInMemory(report = this.reportState): void {\n    report.dump = this.createEmptyDump();\n    report.executionIndexByRunner = new WeakMap();\n    report.lastExecution = undefined;\n    report.yamlRecords.length = 0;\n  }\n\n  private assertReportIdle(): void {\n    assert(\n      !this.reportState.activeOperations.size &&\n        !this.context.isExternallyBusy(),\n      'Cannot reset or finalize a report while the Agent is executing. Await the active operation first.',\n    );\n  }\n\n  assertReportWritable(): AgentReportState {\n    const report = this.reportState;\n    assert(\n      !this.context.isDestroyed(),\n      'Cannot execute using a destroyed Agent.',\n    );\n    assert(\n      !this.reportReset,\n      'Report reset is in progress. Await resetDump() first.',\n    );\n    assert(\n      !report.finalization,\n      'The report has been finalized. Call and await resetDump() before executing again.',\n    );\n    return report;\n  }\n\n  async withReportOperation<T>(operation: () => Promise<T>): Promise<T> {\n    const report = this.assertReportWritable();\n    const pending = Promise.resolve().then(operation);\n    report.activeOperations.add(pending);\n    try {\n      return await pending;\n    } finally {\n      report.activeOperations.delete(pending);\n    }\n  }\n\n  /**\n   * Finalize the preceding report and reset execution records for a new report.\n   * Historical artifacts are retained; this Agent and its device stay connected.\n   * Await this method before executing again. Concurrent resets share one reset.\n   */\n  async resetDump(): Promise<ReportActionDump> {\n    assert(\n      !this.context.isDestroyed(),\n      'Cannot reset a report using a destroyed Agent.',\n    );\n    if (this.reportReset) return this.reportReset;\n    this.assertReportIdle();\n    this.reportReset = (async () => {\n      try {\n        await this.finalizeReport();\n      } catch (error) {\n        // Failure belongs to the preceding report, not to the next write queue.\n        warn(\n          `Failed to finalize previous report; resetting dump: ${String(error)}`,\n        );\n      }\n      assert(\n        !this.context.isDestroyed(),\n        'Cannot reset a report using a destroyed Agent.',\n      );\n      const fileName = getReportFileName(\n        this.opts.testId || this.context.getDeviceType() || 'web',\n      );\n      this.reportState = this.createReportState(fileName);\n      return this.dump;\n    })();\n    try {\n      return await this.reportReset;\n    } finally {\n      this.reportReset = undefined;\n    }\n  }\n\n  /**\n   * Seal the current report and release its in-memory records without\n   * disconnecting the device. Export in-memory data before calling this method.\n   * Repeated calls return the same result and never rewrite published artifacts.\n   */\n  async finalizeReport(): Promise<string | undefined> {\n    const report = this.reportState;\n    if (!report.finalization) {\n      this.assertReportIdle();\n      report.finalization = (async () => {\n        await report.generator.flush();\n        const path = await report.generator.finalize();\n        report.file = path;\n        this.clearDumpInMemory(report);\n        return path;\n      })();\n    }\n    return report.finalization;\n  }\n\n  appendExecutionDump(execution: ExecutionDump, runner?: TaskRunner) {\n    assert(\n      !this.reportState.finalization,\n      'Cannot append to a finalized report. Await resetDump() first.',\n    );\n    const currentDump = this.dump;\n    if (runner) {\n      const existingIndex = this.reportState.executionIndexByRunner.get(runner);\n      if (existingIndex !== undefined) {\n        currentDump.executions[existingIndex] = execution;\n        return;\n      }\n      currentDump.executions.push(execution);\n      this.reportState.executionIndexByRunner.set(\n        runner,\n        currentDump.executions.length - 1,\n      );\n      return;\n    }\n    currentDump.executions.push(execution);\n  }\n\n  dumpDataString(opt?: { inlineScreenshots?: boolean }) {\n    // update dump info\n    this.dump.groupName = this.opts.groupName!;\n    this.dump.groupDescription = this.opts.groupDescription;\n    // In browser environment, use inline screenshots since file system is not available\n    if (ifInBrowser || opt?.inlineScreenshots) {\n      return this.dump.serializeWithInlineScreenshots();\n    }\n    return this.dump.serialize();\n  }\n\n  reportHTMLString(getDumpString: () => string) {\n    // Short-circuit at the call site because JavaScript evaluates function\n    // arguments first. This avoids serializing the dump (including inline\n    // screenshots) when the Report Viewer build does not need report HTML.\n    if (IS_REPORT_BUILD) {\n      return '';\n    }\n\n    // dumpDataString() handles browser environment with inline screenshots\n    const dumpString = getDumpString();\n    if (!this.reportState.yamlRecords.length)\n      return reportHTMLContent(dumpString);\n\n    // Browser exports have no file writer, so the execution record must own\n    // the hierarchy independently of whether file report generation is enabled.\n    const reportId = this.reportState.yamlScopeId;\n    const runnerDump = buildAgentTestRunReportDump(\n      this.buildYamlReportInput(),\n      this.dump,\n      {\n        reportId,\n        scopeId: this.reportState.yamlScopeId,\n        sourcePath: this.reportFile ?? '<inline-agent-report>',\n      },\n    );\n    return `${reportHTMLContent({\n      dumpString,\n      attributes: { 'data-group-id': reportId, 'data-report-id': reportId },\n    })}\\n${generateTestRunReportScriptTag(runnerDump)}`;\n  }\n\n  writeOutActionDumps(executionDump?: ExecutionDump) {\n    const report = this.reportState;\n    if (report.finalization) {\n      if (executionDump) {\n        throw new WorkflowError(\n          'Cannot append to a finalized report. Await resetDump() first.',\n          {\n            code: 'REPORT_FINALIZED',\n          },\n        );\n      }\n      return;\n    }\n    const exec = executionDump || report.lastExecution;\n    if (exec) {\n      report.lastExecution = exec;\n      report.generator.onExecutionUpdate(\n        exec,\n        this.getReportMeta(),\n        this.opts.reportAttributes,\n      );\n    }\n    report.file = report.generator.getReportPath();\n  }\n\n  /** Persist current report data without disposing this Agent or its device. */\n  async flushReport(): Promise<string | undefined> {\n    if (this.reportReset) await this.reportReset;\n    const report = this.reportState;\n    if (report.finalization) return report.finalization;\n    this.writeOutActionDumps();\n    await report.generator.flush();\n    return report.lastExecution ? (report.file ?? undefined) : undefined;\n  }\n\n  private getReportMeta(): ReportMeta {\n    return {\n      groupName: this.dump.groupName,\n      groupDescription: this.dump.groupDescription,\n      sdkVersion: this.dump.sdkVersion,\n      modelBriefs: this.dump.modelBriefs,\n      deviceType: this.dump.deviceType,\n    };\n  }\n\n  get onDumpUpdate():\n    | ((dump: string, executionDump?: ExecutionDump) => void)\n    | undefined {\n    return this.dumpUpdateListeners[0];\n  }\n\n  set onDumpUpdate(callback:\n    | ((dump: string, executionDump?: ExecutionDump) => void)\n    | undefined) {\n    // Clear existing listeners\n    this.dumpUpdateListeners = [];\n    // Add callback to array if provided\n    if (callback) {\n      this.dumpUpdateListeners.push(callback);\n    }\n  }\n\n  addDumpUpdateListener(\n    listener: (dump: string, executionDump?: ExecutionDump) => void,\n  ): () => void {\n    this.dumpUpdateListeners.push(listener);\n\n    // Return remove function\n    return () => {\n      this.removeDumpUpdateListener(listener);\n    };\n  }\n\n  /**\n   * Remove a dump update listener\n   * @param listener The listener function to remove\n   */\n  removeDumpUpdateListener(\n    listener: (dump: string, executionDump?: ExecutionDump) => void,\n  ): void {\n    const index = this.dumpUpdateListeners.indexOf(listener);\n    if (index > -1) {\n      this.dumpUpdateListeners.splice(index, 1);\n    }\n  }\n\n  /**\n   * Clear all dump update listeners\n   */\n  clearDumpUpdateListeners(): void {\n    this.dumpUpdateListeners = [];\n  }\n\n  notifyDumpUpdateListeners(executionDump?: ExecutionDump) {\n    if (this.dumpUpdateListeners.length === 0) return;\n    const dumpString = this.dumpDataString();\n    for (const listener of this.dumpUpdateListeners) {\n      try {\n        listener(dumpString, executionDump);\n      } catch (error) {\n        console.error('Error in onDumpUpdate listener', error);\n      }\n    }\n  }\n\n  bindRunner(runner: TaskRunner): void {\n    this.reportByRunner.set(runner, this.reportState);\n  }\n\n  assertRunnerCurrent(runner: TaskRunner): void {\n    const report = this.reportByRunner.get(runner);\n    assert(\n      report === this.reportState && !report.finalization,\n      'Cannot write a late execution update into a different or finalized report.',\n    );\n  }\n\n  async flushPendingWrites(): Promise<void> {\n    await this.reportState.generator.flush();\n  }\n\n  async waitForOperations(): Promise<void> {\n    while (this.reportState.activeOperations.size) {\n      await Promise.allSettled([...this.reportState.activeOperations]);\n    }\n  }\n\n  async waitForReset(): Promise<void> {\n    if (this.reportReset) await this.reportReset.catch(() => {});\n  }\n}\n"],"names":["warn","getDebug","AgentReportManager","value","report","session","currentReportSession","Error","recorder","undefined","scopeId","executionIds","record","existingIndex","entry","recordIndex","path","error","Object","scopedRecords","outcome","attempt","executionRecordsToReportInput","ReportActionDump","getVersion","fileName","reuseExistingReport","assertReportGenerationOptions","state","nullReportGenerator","ReportGenerator","WeakMap","uuid","Set","assert","operation","pending","Promise","String","getReportFileName","execution","runner","currentDump","opt","ifInBrowser","getDumpString","IS_REPORT_BUILD","dumpString","reportHTMLContent","reportId","runnerDump","buildAgentTestRunReportDump","generateTestRunReportScriptTag","executionDump","WorkflowError","exec","callback","listener","index","console","context","opts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAMA,OAAOC,SAAS,SAAS;IAAE,SAAS;AAAK;AAqCxC,MAAMC;IAmBX,IAAY,OAAsB;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU;IAChC;IAEA,IAAI,OAAyB;QAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI;IAC9B;IACA,IAAI,KAAKC,KAAuB,EAAE;QAChC,IAAI,CAAC,WAAW,CAAC,IAAI,GAAGA;IAC1B;IAEA,IAAI,aAAwC;QAC1C,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI;IAC9B;IACA,IAAI,WAAWA,KAAgC,EAAE;QAC/C,IAAI,CAAC,WAAW,CAAC,IAAI,GAAGA;IAC1B;IAEA,IAAI,iBAAqC;QACvC,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ;IAClC;IACA,IAAI,eAAeA,KAAyB,EAAE;QAC5C,IAAI,CAAC,WAAW,CAAC,QAAQ,GAAGA;IAC9B;IAGA,MAAM,wBAAuC;QAC3C,MAAMC,SAAS,IAAI,CAAC,WAAW;QAC/B,MAAMC,UAAUC;QAChB,IAAID,WAAWA,YAAYD,OAAO,OAAO,EAAE;YACzC,IAAIA,OAAO,OAAO,EAChB,MAAM,IAAIG,MACR;YAEJ,IAAI,CAAC,gBAAgB;YACrB,MAAMH,OAAO,SAAS,CAAC,QAAQ;YAC/B,MAAMI,WAAWH,QAAQ,cAAc,CAAC,IAAM,IAAI,CAAC,YAAY,CAACD;YAGhE,IAAI,CAAC,iBAAiB,CAACA;YACvBA,OAAO,SAAS,GAAGI;YACnBJ,OAAO,OAAO,GAAGC;YACjBD,OAAO,IAAI,GAAGK;QAChB;QACAL,OAAO,SAAS,CAAC,qBAAqB;IACxC;IACA,MAAM,oBAAoBM,OAAe,EAAEC,YAAgC,EAAE;QAC3E,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,oBAAoB,GACpDD,SACAC;IAEJ;IAGA,MAAM,0BACJC,MAA+B,EACF;QAC7B,MAAMR,SAAS,IAAI,CAAC,WAAW;QAC/B,MAAMS,gBAAgBT,OAAO,WAAW,CAAC,SAAS,CAChD,CAACU,QAAUA,MAAM,KAAK,KAAKF,OAAO,KAAK;QAEzC,MAAMG,cACJF,gBAAgB,IAAIT,OAAO,WAAW,CAAC,MAAM,GAAGS;QAClDT,OAAO,WAAW,CAACW,YAAY,GAAGH;QAClC,IAAI;YACF,MAAMI,OAAO,MAAMZ,OAAO,SAAS,CAAC,iBAAiB,GACnD,IAAI,CAAC,oBAAoB,IACzBA,OAAO,WAAW,EAClB,IAAI,CAAC,aAAa;YAEpBA,OAAO,IAAI,GAAGY,QAAQZ,OAAO,IAAI;YACjC,OAAOY;QACT,EAAE,OAAOC,OAAO;YACdb,OAAO,WAAW,CAACW,YAAY,GAAGG,OAAO,MAAM,CAAC;gBAC9C,GAAGN,MAAM;gBACT,QAAQ;gBACR,aAAaK;YACf;YACA,MAAMA;QACR;IACF;IAEQ,uBAAuB;QAC7B,MAAME,gBAAgB,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAACL,QAAW;gBACjE,GAAGA,KAAK;gBACR,GAAIA,MAAM,SAAS,GACf;oBACE,WAAW;wBACT,UAAU;4BACR,GAAGA,MAAM,SAAS,CAAC,QAAQ;4BAC3B,eAAe,IAAI,CAAC,WAAW,CAAC,WAAW;wBAC7C;wBACA,OAAOA,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAACM,UAAa;gCAC7C,GAAGA,OAAO;gCACV,GAAIA,QAAQ,GAAG,GACX;oCACE,KAAK;wCACH,GAAGA,QAAQ,GAAG;wCACd,eAAe,IAAI,CAAC,WAAW,CAAC,WAAW;oCAC7C;gCACF,IACA,CAAC,CAAC;gCACN,GAAIA,QAAQ,QAAQ,GAChB;oCACE,UAAUA,QAAQ,QAAQ,CAAC,GAAG,CAAC,CAACC,UAAa;4CAC3C,GAAGA,OAAO;4CACV,eAAe,IAAI,CAAC,WAAW,CAAC,WAAW;wCAC7C;gCACF,IACA,CAAC,CAAC;4BACR;oBACF;gBACF,IACA,CAAC,CAAC;YACR;QACA,OAAOC,8BAA8BH,eAAe;YAClD,OAAO,IAAI,CAAC,WAAW,CAAC,WAAW;QACrC;IACF;IAEQ,kBAAoC;QAC1C,OAAO,IAAII,iBAAiB;YAC1B,YAAYC;YACZ,WAAW,IAAI,CAAC,IAAI,CAAC,SAAS;YAC9B,kBAAkB,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAC5C,YAAY,EAAE;YACd,aAAa,EAAE;YACf,YAAY,IAAI,CAAC,OAAO,CAAC,aAAa;QACxC;IACF;IAEQ,kBACNC,QAAgB,EAChBC,sBAAsB,KAAK,EACT;QAClBC,8BAA8B,IAAI,CAAC,IAAI;QACvC,MAAMtB,UAAUC;QAChB,MAAMsB,QAA0B;YAC9B,MAAM,IAAI,CAAC,eAAe;YAC1BH;YACA,WAAWpB,UACPwB,sBACAC,gBAAgB,MAAM,CAACL,UAAU;gBAC/B,gBAAgB,IAAI,CAAC,IAAI,CAAC,cAAc;gBACxC,sBAAsB,IAAI,CAAC,IAAI,CAAC,oBAAoB;gBACpD,cAAc,IAAI,CAAC,IAAI,CAAC,YAAY;gBACpC,oBAAoB,IAAI,CAAC,IAAI,CAAC,kBAAkB;gBAChDC;YACF;YACJ,wBAAwB,IAAIK;YAC5B,aAAaC;YACb,aAAa,EAAE;YACf,kBAAkB,IAAIC;QACxB;QACA,IAAI5B,SAAS;YACXuB,MAAM,SAAS,GAAGvB,QAAQ,cAAc,CAAC,IAAM,IAAI,CAAC,YAAY,CAACuB;YACjEA,MAAM,OAAO,GAAGvB;QAClB;QACA,OAAOuB;IACT;IAEQ,aAAaxB,MAAwB,EAAW;QACtD,OAAOA,OAAO,gBAAgB,CAAC,IAAI,GAAG,KAAK,IAAI,CAAC,OAAO,CAAC,gBAAgB;IAC1E;IAEQ,kBAAkBA,SAAS,IAAI,CAAC,WAAW,EAAQ;QACzDA,OAAO,IAAI,GAAG,IAAI,CAAC,eAAe;QAClCA,OAAO,sBAAsB,GAAG,IAAI2B;QACpC3B,OAAO,aAAa,GAAGK;QACvBL,OAAO,WAAW,CAAC,MAAM,GAAG;IAC9B;IAEQ,mBAAyB;QAC/B8B,OACE,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,IAAI,IACrC,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,IAChC;IAEJ;IAEA,uBAAyC;QACvC,MAAM9B,SAAS,IAAI,CAAC,WAAW;QAC/B8B,OACE,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,IACzB;QAEFA,OACE,CAAC,IAAI,CAAC,WAAW,EACjB;QAEFA,OACE,CAAC9B,OAAO,YAAY,EACpB;QAEF,OAAOA;IACT;IAEA,MAAM,oBAAuB+B,SAA2B,EAAc;QACpE,MAAM/B,SAAS,IAAI,CAAC,oBAAoB;QACxC,MAAMgC,UAAUC,QAAQ,OAAO,GAAG,IAAI,CAACF;QACvC/B,OAAO,gBAAgB,CAAC,GAAG,CAACgC;QAC5B,IAAI;YACF,OAAO,MAAMA;QACf,SAAU;YACRhC,OAAO,gBAAgB,CAAC,MAAM,CAACgC;QACjC;IACF;IAOA,MAAM,YAAuC;QAC3CF,OACE,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,IACzB;QAEF,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,IAAI,CAAC,WAAW;QAC7C,IAAI,CAAC,gBAAgB;QACrB,IAAI,CAAC,WAAW,GAAI;YAClB,IAAI;gBACF,MAAM,IAAI,CAAC,cAAc;YAC3B,EAAE,OAAOjB,OAAO;gBAEdjB,KACE,CAAC,oDAAoD,EAAEsC,OAAOrB,QAAQ;YAE1E;YACAiB,OACE,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,IACzB;YAEF,MAAMT,WAAWc,kBACf,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,MAAM;YAEtD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAACd;YAC1C,OAAO,IAAI,CAAC,IAAI;QAClB;QACA,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,WAAW;QAC/B,SAAU;YACR,IAAI,CAAC,WAAW,GAAGhB;QACrB;IACF;IAOA,MAAM,iBAA8C;QAClD,MAAML,SAAS,IAAI,CAAC,WAAW;QAC/B,IAAI,CAACA,OAAO,YAAY,EAAE;YACxB,IAAI,CAAC,gBAAgB;YACrBA,OAAO,YAAY,GAAI;gBACrB,MAAMA,OAAO,SAAS,CAAC,KAAK;gBAC5B,MAAMY,OAAO,MAAMZ,OAAO,SAAS,CAAC,QAAQ;gBAC5CA,OAAO,IAAI,GAAGY;gBACd,IAAI,CAAC,iBAAiB,CAACZ;gBACvB,OAAOY;YACT;QACF;QACA,OAAOZ,OAAO,YAAY;IAC5B;IAEA,oBAAoBoC,SAAwB,EAAEC,MAAmB,EAAE;QACjEP,OACE,CAAC,IAAI,CAAC,WAAW,CAAC,YAAY,EAC9B;QAEF,MAAMQ,cAAc,IAAI,CAAC,IAAI;QAC7B,IAAID,QAAQ;YACV,MAAM5B,gBAAgB,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAAC,GAAG,CAAC4B;YAClE,IAAI5B,AAAkBJ,WAAlBI,eAA6B;gBAC/B6B,YAAY,UAAU,CAAC7B,cAAc,GAAG2B;gBACxC;YACF;YACAE,YAAY,UAAU,CAAC,IAAI,CAACF;YAC5B,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAAC,GAAG,CACzCC,QACAC,YAAY,UAAU,CAAC,MAAM,GAAG;YAElC;QACF;QACAA,YAAY,UAAU,CAAC,IAAI,CAACF;IAC9B;IAEA,eAAeG,GAAqC,EAAE;QAEpD,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;QAEvD,IAAIC,eAAeD,KAAK,mBACtB,OAAO,IAAI,CAAC,IAAI,CAAC,8BAA8B;QAEjD,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS;IAC5B;IAEA,iBAAiBE,aAA2B,EAAE;QAI5C,IAAIC,iBACF,OAAO;QAIT,MAAMC,aAAaF;QACnB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,EACtC,OAAOG,kBAAkBD;QAI3B,MAAME,WAAW,IAAI,CAAC,WAAW,CAAC,WAAW;QAC7C,MAAMC,aAAaC,4BACjB,IAAI,CAAC,oBAAoB,IACzB,IAAI,CAAC,IAAI,EACT;YACEF;YACA,SAAS,IAAI,CAAC,WAAW,CAAC,WAAW;YACrC,YAAY,IAAI,CAAC,UAAU,IAAI;QACjC;QAEF,OAAO,GAAGD,kBAAkB;YAC1BD;YACA,YAAY;gBAAE,iBAAiBE;gBAAU,kBAAkBA;YAAS;QACtE,GAAG,EAAE,EAAEG,+BAA+BF,aAAa;IACrD;IAEA,oBAAoBG,aAA6B,EAAE;QACjD,MAAMjD,SAAS,IAAI,CAAC,WAAW;QAC/B,IAAIA,OAAO,YAAY,EAAE;YACvB,IAAIiD,eACF,MAAM,IAAIC,cACR,iEACA;gBACE,MAAM;YACR;YAGJ;QACF;QACA,MAAMC,OAAOF,iBAAiBjD,OAAO,aAAa;QAClD,IAAImD,MAAM;YACRnD,OAAO,aAAa,GAAGmD;YACvBnD,OAAO,SAAS,CAAC,iBAAiB,CAChCmD,MACA,IAAI,CAAC,aAAa,IAClB,IAAI,CAAC,IAAI,CAAC,gBAAgB;QAE9B;QACAnD,OAAO,IAAI,GAAGA,OAAO,SAAS,CAAC,aAAa;IAC9C;IAGA,MAAM,cAA2C;QAC/C,IAAI,IAAI,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,WAAW;QAC5C,MAAMA,SAAS,IAAI,CAAC,WAAW;QAC/B,IAAIA,OAAO,YAAY,EAAE,OAAOA,OAAO,YAAY;QACnD,IAAI,CAAC,mBAAmB;QACxB,MAAMA,OAAO,SAAS,CAAC,KAAK;QAC5B,OAAOA,OAAO,aAAa,GAAIA,OAAO,IAAI,IAAIK,SAAaA;IAC7D;IAEQ,gBAA4B;QAClC,OAAO;YACL,WAAW,IAAI,CAAC,IAAI,CAAC,SAAS;YAC9B,kBAAkB,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAC5C,YAAY,IAAI,CAAC,IAAI,CAAC,UAAU;YAChC,aAAa,IAAI,CAAC,IAAI,CAAC,WAAW;YAClC,YAAY,IAAI,CAAC,IAAI,CAAC,UAAU;QAClC;IACF;IAEA,IAAI,eAEU;QACZ,OAAO,IAAI,CAAC,mBAAmB,CAAC,EAAE;IACpC;IAEA,IAAI,aAAa+C,QAEJ,EAAE;QAEb,IAAI,CAAC,mBAAmB,GAAG,EAAE;QAE7B,IAAIA,UACF,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAACA;IAElC;IAEA,sBACEC,QAA+D,EACnD;QACZ,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAACA;QAG9B,OAAO;YACL,IAAI,CAAC,wBAAwB,CAACA;QAChC;IACF;IAMA,yBACEA,QAA+D,EACzD;QACN,MAAMC,QAAQ,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAACD;QAC/C,IAAIC,QAAQ,IACV,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAACA,OAAO;IAE3C;IAKA,2BAAiC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,EAAE;IAC/B;IAEA,0BAA0BL,aAA6B,EAAE;QACvD,IAAI,AAAoC,MAApC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAQ;QAC3C,MAAMN,aAAa,IAAI,CAAC,cAAc;QACtC,KAAK,MAAMU,YAAY,IAAI,CAAC,mBAAmB,CAC7C,IAAI;YACFA,SAASV,YAAYM;QACvB,EAAE,OAAOpC,OAAO;YACd0C,QAAQ,KAAK,CAAC,kCAAkC1C;QAClD;IAEJ;IAEA,WAAWwB,MAAkB,EAAQ;QACnC,IAAI,CAAC,cAAc,CAAC,GAAG,CAACA,QAAQ,IAAI,CAAC,WAAW;IAClD;IAEA,oBAAoBA,MAAkB,EAAQ;QAC5C,MAAMrC,SAAS,IAAI,CAAC,cAAc,CAAC,GAAG,CAACqC;QACvCP,OACE9B,WAAW,IAAI,CAAC,WAAW,IAAI,CAACA,OAAO,YAAY,EACnD;IAEJ;IAEA,MAAM,qBAAoC;QACxC,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK;IACxC;IAEA,MAAM,oBAAmC;QACvC,MAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAC3C,MAAMiC,QAAQ,UAAU,CAAC;eAAI,IAAI,CAAC,WAAW,CAAC,gBAAgB;SAAC;IAEnE;IAEA,MAAM,eAA8B;QAClC,IAAI,IAAI,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,KAAO;IAC5D;IAtdA,YAA6BuB,OAA2B,CAAE;;QAP1D,uBAAQ,eAAR;QACA,uBAAQ,eAAR;QACA,uBAAiB,kBAAjB;QACA,uBAAQ,uBAAR;aAI6BA,OAAO,GAAPA;aALZ,cAAc,GAAG,IAAI7B;aAC9B,mBAAmB,GAEvB,EAAE;QAGJ,MAAM8B,OAAO,IAAI,CAAC,IAAI;QACtB,MAAMpC,WACJoC,KAAK,cAAc,IACnBtB,kBAAkBsB,KAAK,MAAM,IAAID,QAAQ,aAAa,MAAM;QAC9D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,iBAAiB,CACvCnC,UACAoC,KAAK,gBAAgB,EAAE,CAAC,gBAAgB,KAAKpC;IAEjD;AA8cF"}
+{"version":3,"file":"agent/agent-report-manager.mjs","sources":["../../../src/agent/agent-report-manager.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert, ifInBrowser, uuid } from '@midscene/shared/utils';\nimport { IS_REPORT_BUILD } from '../constants';\nimport { generateTestRunReportScriptTag } from '../dump/html-utils';\nimport {\n  type ReportSession,\n  currentReportSession,\n} from '../dump/report-session';\nimport {\n  type IReportGenerator,\n  ReportGenerator,\n  assertReportGenerationOptions,\n  nullReportGenerator,\n} from '../report-generator';\nimport type { TaskRunner } from '../task-runner';\nimport { WorkflowError } from '../test-runner/errors';\nimport type { WorkflowExecutionRecord } from '../test-runner/execution-record';\nimport { buildAgentTestRunReportDump } from '../test-runner/reporting/agent-report';\nimport { executionRecordsToReportInput } from '../test-runner/reporting/execution-record';\nimport {\n  type AgentOpt,\n  type ExecutionDump,\n  ReportActionDump,\n  type ReportMeta,\n} from '../types';\nimport { getVersion, reportHTMLContent } from '../utils';\nimport { getReportFileName } from './utils';\n\nconst warn = getDebug('agent', { console: true });\n\ntype ReportOptions = Pick<\n  AgentOpt,\n  | 'groupName'\n  | 'groupDescription'\n  | 'generateReport'\n  | 'persistExecutionDump'\n  | 'outputFormat'\n  | 'autoPrintReportMsg'\n  | 'reportAttributes'\n  | 'reportFileName'\n  | 'testId'\n>;\n\ninterface AgentReportContext {\n  getOptions(): ReportOptions;\n  getDeviceType(): ReportActionDump['deviceType'];\n  isDestroyed(): boolean;\n  isExternallyBusy(): boolean;\n}\n\ninterface AgentReportState {\n  dump: ReportActionDump;\n  file?: string | null;\n  fileName?: string;\n  generator: IReportGenerator;\n  session?: ReportSession;\n  finalization?: Promise<string | undefined>;\n  lastExecution?: ExecutionDump;\n  executionIndexByRunner: WeakMap<TaskRunner, number>;\n  yamlScopeId: string;\n  yamlRecords: WorkflowExecutionRecord[];\n  activeOperations: Set<Promise<unknown>>;\n}\n\n/** Owns report state and persistence; execution and device ownership stay with Agent. */\nexport class AgentReportManager {\n  private reportState: AgentReportState;\n  private reportReset?: Promise<ReportActionDump>;\n  private readonly reportByRunner = new WeakMap<TaskRunner, AgentReportState>();\n  private dumpUpdateListeners: Array<\n    (dump: string, executionDump?: ExecutionDump) => void\n  > = [];\n\n  constructor(private readonly context: AgentReportContext) {\n    const opts = this.opts;\n    const fileName =\n      opts.reportFileName ??\n      getReportFileName(opts.testId || context.getDeviceType() || 'web');\n    this.reportState = this.createReportState(\n      fileName,\n      opts.reportAttributes?.['data-group-id'] === fileName,\n    );\n  }\n\n  private get opts(): ReportOptions {\n    return this.context.getOptions();\n  }\n\n  get dump(): ReportActionDump {\n    return this.reportState.dump;\n  }\n  set dump(value: ReportActionDump) {\n    this.reportState.dump = value;\n  }\n\n  get reportFile(): string | null | undefined {\n    return this.reportState.file;\n  }\n  set reportFile(value: string | null | undefined) {\n    this.reportState.file = value;\n  }\n\n  get reportFileName(): string | undefined {\n    return this.reportState.fileName;\n  }\n  set reportFileName(value: string | undefined) {\n    this.reportState.fileName = value;\n  }\n\n  /** @internal Runner snapshots retain immutable data and shared image files. */\n  async _prepareForTestRunner(): Promise<void> {\n    const report = this.reportState;\n    const session = currentReportSession();\n    if (session && session !== report.session) {\n      if (report.session)\n        throw new Error(\n          'An Agent cannot be shared by different Test report sessions. Await resetDump() between runs.',\n        );\n      this.assertReportIdle();\n      await report.generator.finalize();\n      const recorder = session.createRecorder(() => this.isReportBusy(report));\n      // The previous standalone report owns its history. The Test stream starts\n      // at acquisition, so previous script operations do not enter this run.\n      this.clearDumpInMemory(report);\n      report.generator = recorder;\n      report.session = session;\n      report.file = undefined;\n    }\n    report.generator.enableSourceSnapshots?.(report.yamlScopeId);\n  }\n  async _createReportSource(scopeId: string, executionIds?: readonly string[]) {\n    return this.reportState.generator.createSourceSnapshot?.(\n      scopeId,\n      executionIds,\n    );\n  }\n\n  /** @internal The Agent owns one report stream across root YAML invocations. */\n  async _writeYamlExecutionReport(\n    record: WorkflowExecutionRecord,\n  ): Promise<string | undefined> {\n    const report = this.reportState;\n    const existingIndex = report.yamlRecords.findIndex(\n      (entry) => entry.runId === record.runId,\n    );\n    const recordIndex =\n      existingIndex < 0 ? report.yamlRecords.length : existingIndex;\n    report.yamlRecords[recordIndex] = record;\n    try {\n      const path = await report.generator.writeRunnerReport?.(\n        this.buildYamlReportInput(),\n        report.yamlScopeId,\n        this.getReportMeta(),\n      );\n      report.file = path ?? report.file;\n      return path;\n    } catch (error) {\n      report.yamlRecords[recordIndex] = Object.freeze({\n        ...record,\n        status: 'failed',\n        reportError: error,\n      });\n      throw error;\n    }\n  }\n\n  private buildYamlReportInput() {\n    const scopedRecords = this.reportState.yamlRecords.map((entry) => ({\n      ...entry,\n      ...(entry.execution\n        ? {\n            execution: {\n              document: {\n                ...entry.execution.document,\n                reportScopeId: this.reportState.yamlScopeId,\n              },\n              cases: entry.execution.cases.map((outcome) => ({\n                ...outcome,\n                ...(outcome.run\n                  ? {\n                      run: {\n                        ...outcome.run,\n                        reportScopeId: this.reportState.yamlScopeId,\n                      },\n                    }\n                  : {}),\n                ...(outcome.attempts\n                  ? {\n                      attempts: outcome.attempts.map((attempt) => ({\n                        ...attempt,\n                        reportScopeId: this.reportState.yamlScopeId,\n                      })),\n                    }\n                  : {}),\n              })),\n            },\n          }\n        : {}),\n    }));\n    return executionRecordsToReportInput(scopedRecords, {\n      runId: this.reportState.yamlScopeId,\n    });\n  }\n\n  private createEmptyDump(): ReportActionDump {\n    return new ReportActionDump({\n      sdkVersion: getVersion(),\n      groupName: this.opts.groupName!,\n      groupDescription: this.opts.groupDescription,\n      executions: [],\n      modelBriefs: [],\n      deviceType: this.context.getDeviceType(),\n    });\n  }\n\n  private createReportState(\n    fileName: string,\n    reuseExistingReport = false,\n  ): AgentReportState {\n    assertReportGenerationOptions(this.opts);\n    const session = currentReportSession();\n    const state: AgentReportState = {\n      dump: this.createEmptyDump(),\n      fileName,\n      generator: session\n        ? nullReportGenerator\n        : ReportGenerator.create(fileName, {\n            generateReport: this.opts.generateReport,\n            persistExecutionDump: this.opts.persistExecutionDump,\n            outputFormat: this.opts.outputFormat,\n            autoPrintReportMsg: this.opts.autoPrintReportMsg,\n            reuseExistingReport,\n          }),\n      executionIndexByRunner: new WeakMap(),\n      yamlScopeId: uuid(),\n      yamlRecords: [],\n      activeOperations: new Set(),\n    };\n    if (session) {\n      state.generator = session.createRecorder(() => this.isReportBusy(state));\n      state.session = session;\n    }\n    return state;\n  }\n\n  private isReportBusy(report: AgentReportState): boolean {\n    return report.activeOperations.size > 0 || this.context.isExternallyBusy();\n  }\n\n  private clearDumpInMemory(report = this.reportState): void {\n    report.dump = this.createEmptyDump();\n    report.executionIndexByRunner = new WeakMap();\n    report.lastExecution = undefined;\n    report.yamlRecords.length = 0;\n  }\n\n  private assertReportIdle(): void {\n    assert(\n      !this.reportState.activeOperations.size &&\n        !this.context.isExternallyBusy(),\n      'Cannot reset or finalize a report while the Agent is executing. Await the active operation first.',\n    );\n  }\n\n  assertReportWritable(): AgentReportState {\n    const report = this.reportState;\n    assert(\n      !this.context.isDestroyed(),\n      'Cannot execute using a destroyed Agent.',\n    );\n    assert(\n      !this.reportReset,\n      'Report reset is in progress. Await resetDump() first.',\n    );\n    assert(\n      !report.finalization,\n      'The report has been finalized. Call and await resetDump() before executing again.',\n    );\n    return report;\n  }\n\n  async withReportOperation<T>(operation: () => Promise<T>): Promise<T> {\n    const report = this.assertReportWritable();\n    const pending = Promise.resolve().then(operation);\n    report.activeOperations.add(pending);\n    try {\n      return await pending;\n    } finally {\n      report.activeOperations.delete(pending);\n    }\n  }\n\n  /**\n   * Finalize the preceding report and reset execution records for a new report.\n   * Historical artifacts are retained; this Agent and its device stay connected.\n   * Await this method before executing again. Concurrent resets share one reset.\n   */\n  async resetDump(): Promise<ReportActionDump> {\n    assert(\n      !this.context.isDestroyed(),\n      'Cannot reset a report using a destroyed Agent.',\n    );\n    if (this.reportReset) return this.reportReset;\n    this.assertReportIdle();\n    this.reportReset = (async () => {\n      try {\n        await this.finalizeReport();\n      } catch (error) {\n        // Failure belongs to the preceding report, not to the next write queue.\n        warn(\n          `Failed to finalize previous report; resetting dump: ${String(error)}`,\n        );\n      }\n      assert(\n        !this.context.isDestroyed(),\n        'Cannot reset a report using a destroyed Agent.',\n      );\n      const fileName = getReportFileName(\n        this.opts.testId || this.context.getDeviceType() || 'web',\n      );\n      this.reportState = this.createReportState(fileName);\n      return this.dump;\n    })();\n    try {\n      return await this.reportReset;\n    } finally {\n      this.reportReset = undefined;\n    }\n  }\n\n  /**\n   * Seal the current report and release its in-memory records without\n   * disconnecting the device. Export in-memory data before calling this method.\n   * Repeated calls return the same result and never rewrite published artifacts.\n   */\n  async finalizeReport(): Promise<string | undefined> {\n    const report = this.reportState;\n    if (!report.finalization) {\n      this.assertReportIdle();\n      report.finalization = (async () => {\n        await report.generator.flush();\n        const path = await report.generator.finalize();\n        report.file = path;\n        this.clearDumpInMemory(report);\n        return path;\n      })();\n    }\n    return report.finalization;\n  }\n\n  appendExecutionDump(execution: ExecutionDump, runner?: TaskRunner) {\n    assert(\n      !this.reportState.finalization,\n      'Cannot append to a finalized report. Await resetDump() first.',\n    );\n    const currentDump = this.dump;\n    if (runner) {\n      const existingIndex = this.reportState.executionIndexByRunner.get(runner);\n      if (existingIndex !== undefined) {\n        currentDump.executions[existingIndex] = execution;\n        return;\n      }\n      currentDump.executions.push(execution);\n      this.reportState.executionIndexByRunner.set(\n        runner,\n        currentDump.executions.length - 1,\n      );\n      return;\n    }\n    currentDump.executions.push(execution);\n  }\n\n  dumpDataString(opt?: { inlineScreenshots?: boolean }) {\n    // update dump info\n    this.dump.groupName = this.opts.groupName!;\n    this.dump.groupDescription = this.opts.groupDescription;\n    // In browser environment, use inline screenshots since file system is not available\n    if (ifInBrowser || opt?.inlineScreenshots) {\n      return this.dump.serializeWithInlineScreenshots();\n    }\n    return this.dump.serialize();\n  }\n\n  reportHTMLString(getDumpString: () => string) {\n    // Short-circuit at the call site because JavaScript evaluates function\n    // arguments first. This avoids serializing the dump (including inline\n    // screenshots) when the Report Viewer build does not need report HTML.\n    if (IS_REPORT_BUILD) {\n      return '';\n    }\n\n    // dumpDataString() handles browser environment with inline screenshots\n    const dumpString = getDumpString();\n    if (!this.reportState.yamlRecords.length)\n      return reportHTMLContent(dumpString);\n\n    // Browser exports have no file writer, so the execution record must own\n    // the hierarchy independently of whether file report generation is enabled.\n    const reportId = this.reportState.yamlScopeId;\n    const runnerDump = buildAgentTestRunReportDump(\n      this.buildYamlReportInput(),\n      this.dump,\n      {\n        reportId,\n        scopeId: this.reportState.yamlScopeId,\n        sourcePath: this.reportFile ?? '<inline-agent-report>',\n      },\n    );\n    return `${reportHTMLContent({\n      dumpString,\n      attributes: { 'data-group-id': reportId, 'data-report-id': reportId },\n    })}\\n${generateTestRunReportScriptTag(runnerDump)}`;\n  }\n\n  writeOutActionDumps(executionDump?: ExecutionDump) {\n    const report = this.reportState;\n    if (report.finalization) {\n      if (executionDump) {\n        throw new WorkflowError(\n          'Cannot append to a finalized report. Await resetDump() first.',\n          {\n            code: 'REPORT_FINALIZED',\n          },\n        );\n      }\n      return;\n    }\n    const exec = executionDump || report.lastExecution;\n    if (exec) {\n      report.lastExecution = exec;\n      report.generator.onExecutionUpdate(\n        exec,\n        this.getReportMeta(),\n        this.opts.reportAttributes,\n      );\n    }\n    report.file = report.generator.getReportPath();\n  }\n\n  /** Persist current report data without disposing this Agent or its device. */\n  async flushReport(): Promise<string | undefined> {\n    if (this.reportReset) await this.reportReset;\n    const report = this.reportState;\n    if (report.finalization) return report.finalization;\n    this.writeOutActionDumps();\n    await report.generator.flush();\n    return report.lastExecution ? (report.file ?? undefined) : undefined;\n  }\n\n  private getReportMeta(): ReportMeta {\n    return {\n      groupName: this.dump.groupName,\n      groupDescription: this.dump.groupDescription,\n      sdkVersion: this.dump.sdkVersion,\n      modelBriefs: this.dump.modelBriefs,\n      deviceType: this.dump.deviceType,\n    };\n  }\n\n  get onDumpUpdate():\n    | ((dump: string, executionDump?: ExecutionDump) => void)\n    | undefined {\n    return this.dumpUpdateListeners[0];\n  }\n\n  set onDumpUpdate(callback:\n    | ((dump: string, executionDump?: ExecutionDump) => void)\n    | undefined) {\n    // Clear existing listeners\n    this.dumpUpdateListeners = [];\n    // Add callback to array if provided\n    if (callback) {\n      this.dumpUpdateListeners.push(callback);\n    }\n  }\n\n  addDumpUpdateListener(\n    listener: (dump: string, executionDump?: ExecutionDump) => void,\n  ): () => void {\n    this.dumpUpdateListeners.push(listener);\n\n    // Return remove function\n    return () => {\n      this.removeDumpUpdateListener(listener);\n    };\n  }\n\n  /**\n   * Remove a dump update listener\n   * @param listener The listener function to remove\n   */\n  removeDumpUpdateListener(\n    listener: (dump: string, executionDump?: ExecutionDump) => void,\n  ): void {\n    const index = this.dumpUpdateListeners.indexOf(listener);\n    if (index > -1) {\n      this.dumpUpdateListeners.splice(index, 1);\n    }\n  }\n\n  /**\n   * Clear all dump update listeners\n   */\n  clearDumpUpdateListeners(): void {\n    this.dumpUpdateListeners = [];\n  }\n\n  notifyDumpUpdateListeners(executionDump?: ExecutionDump) {\n    if (this.dumpUpdateListeners.length === 0) return;\n    const dumpString = this.dumpDataString();\n    for (const listener of this.dumpUpdateListeners) {\n      try {\n        listener(dumpString, executionDump);\n      } catch (error) {\n        console.error('Error in onDumpUpdate listener', error);\n      }\n    }\n  }\n\n  bindRunner(runner: TaskRunner): void {\n    this.reportByRunner.set(runner, this.reportState);\n  }\n\n  assertRunnerCurrent(runner: TaskRunner): void {\n    const report = this.reportByRunner.get(runner);\n    assert(\n      report === this.reportState && !report.finalization,\n      'Cannot write a late execution update into a different or finalized report.',\n    );\n  }\n\n  async flushPendingWrites(): Promise<void> {\n    await this.reportState.generator.flush();\n  }\n\n  async waitForOperations(): Promise<void> {\n    while (this.reportState.activeOperations.size) {\n      await Promise.allSettled([...this.reportState.activeOperations]);\n    }\n  }\n\n  async waitForReset(): Promise<void> {\n    if (this.reportReset) await this.reportReset.catch(() => {});\n  }\n}\n"],"names":["warn","getDebug","AgentReportManager","value","report","session","currentReportSession","Error","recorder","undefined","scopeId","executionIds","record","existingIndex","entry","recordIndex","path","error","Object","scopedRecords","outcome","attempt","executionRecordsToReportInput","ReportActionDump","getVersion","fileName","reuseExistingReport","assertReportGenerationOptions","state","nullReportGenerator","ReportGenerator","WeakMap","uuid","Set","assert","operation","pending","Promise","String","getReportFileName","execution","runner","currentDump","opt","ifInBrowser","getDumpString","IS_REPORT_BUILD","dumpString","reportHTMLContent","reportId","runnerDump","buildAgentTestRunReportDump","generateTestRunReportScriptTag","executionDump","WorkflowError","exec","callback","listener","index","console","context","opts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA4BA,MAAMA,OAAOC,SAAS,SAAS;IAAE,SAAS;AAAK;AAqCxC,MAAMC;IAmBX,IAAY,OAAsB;QAChC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU;IAChC;IAEA,IAAI,OAAyB;QAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI;IAC9B;IACA,IAAI,KAAKC,KAAuB,EAAE;QAChC,IAAI,CAAC,WAAW,CAAC,IAAI,GAAGA;IAC1B;IAEA,IAAI,aAAwC;QAC1C,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI;IAC9B;IACA,IAAI,WAAWA,KAAgC,EAAE;QAC/C,IAAI,CAAC,WAAW,CAAC,IAAI,GAAGA;IAC1B;IAEA,IAAI,iBAAqC;QACvC,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ;IAClC;IACA,IAAI,eAAeA,KAAyB,EAAE;QAC5C,IAAI,CAAC,WAAW,CAAC,QAAQ,GAAGA;IAC9B;IAGA,MAAM,wBAAuC;QAC3C,MAAMC,SAAS,IAAI,CAAC,WAAW;QAC/B,MAAMC,UAAUC;QAChB,IAAID,WAAWA,YAAYD,OAAO,OAAO,EAAE;YACzC,IAAIA,OAAO,OAAO,EAChB,MAAM,IAAIG,MACR;YAEJ,IAAI,CAAC,gBAAgB;YACrB,MAAMH,OAAO,SAAS,CAAC,QAAQ;YAC/B,MAAMI,WAAWH,QAAQ,cAAc,CAAC,IAAM,IAAI,CAAC,YAAY,CAACD;YAGhE,IAAI,CAAC,iBAAiB,CAACA;YACvBA,OAAO,SAAS,GAAGI;YACnBJ,OAAO,OAAO,GAAGC;YACjBD,OAAO,IAAI,GAAGK;QAChB;QACAL,OAAO,SAAS,CAAC,qBAAqB,GAAGA,OAAO,WAAW;IAC7D;IACA,MAAM,oBAAoBM,OAAe,EAAEC,YAAgC,EAAE;QAC3E,OAAO,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,oBAAoB,GACpDD,SACAC;IAEJ;IAGA,MAAM,0BACJC,MAA+B,EACF;QAC7B,MAAMR,SAAS,IAAI,CAAC,WAAW;QAC/B,MAAMS,gBAAgBT,OAAO,WAAW,CAAC,SAAS,CAChD,CAACU,QAAUA,MAAM,KAAK,KAAKF,OAAO,KAAK;QAEzC,MAAMG,cACJF,gBAAgB,IAAIT,OAAO,WAAW,CAAC,MAAM,GAAGS;QAClDT,OAAO,WAAW,CAACW,YAAY,GAAGH;QAClC,IAAI;YACF,MAAMI,OAAO,MAAMZ,OAAO,SAAS,CAAC,iBAAiB,GACnD,IAAI,CAAC,oBAAoB,IACzBA,OAAO,WAAW,EAClB,IAAI,CAAC,aAAa;YAEpBA,OAAO,IAAI,GAAGY,QAAQZ,OAAO,IAAI;YACjC,OAAOY;QACT,EAAE,OAAOC,OAAO;YACdb,OAAO,WAAW,CAACW,YAAY,GAAGG,OAAO,MAAM,CAAC;gBAC9C,GAAGN,MAAM;gBACT,QAAQ;gBACR,aAAaK;YACf;YACA,MAAMA;QACR;IACF;IAEQ,uBAAuB;QAC7B,MAAME,gBAAgB,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAACL,QAAW;gBACjE,GAAGA,KAAK;gBACR,GAAIA,MAAM,SAAS,GACf;oBACE,WAAW;wBACT,UAAU;4BACR,GAAGA,MAAM,SAAS,CAAC,QAAQ;4BAC3B,eAAe,IAAI,CAAC,WAAW,CAAC,WAAW;wBAC7C;wBACA,OAAOA,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAACM,UAAa;gCAC7C,GAAGA,OAAO;gCACV,GAAIA,QAAQ,GAAG,GACX;oCACE,KAAK;wCACH,GAAGA,QAAQ,GAAG;wCACd,eAAe,IAAI,CAAC,WAAW,CAAC,WAAW;oCAC7C;gCACF,IACA,CAAC,CAAC;gCACN,GAAIA,QAAQ,QAAQ,GAChB;oCACE,UAAUA,QAAQ,QAAQ,CAAC,GAAG,CAAC,CAACC,UAAa;4CAC3C,GAAGA,OAAO;4CACV,eAAe,IAAI,CAAC,WAAW,CAAC,WAAW;wCAC7C;gCACF,IACA,CAAC,CAAC;4BACR;oBACF;gBACF,IACA,CAAC,CAAC;YACR;QACA,OAAOC,8BAA8BH,eAAe;YAClD,OAAO,IAAI,CAAC,WAAW,CAAC,WAAW;QACrC;IACF;IAEQ,kBAAoC;QAC1C,OAAO,IAAII,iBAAiB;YAC1B,YAAYC;YACZ,WAAW,IAAI,CAAC,IAAI,CAAC,SAAS;YAC9B,kBAAkB,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAC5C,YAAY,EAAE;YACd,aAAa,EAAE;YACf,YAAY,IAAI,CAAC,OAAO,CAAC,aAAa;QACxC;IACF;IAEQ,kBACNC,QAAgB,EAChBC,sBAAsB,KAAK,EACT;QAClBC,8BAA8B,IAAI,CAAC,IAAI;QACvC,MAAMtB,UAAUC;QAChB,MAAMsB,QAA0B;YAC9B,MAAM,IAAI,CAAC,eAAe;YAC1BH;YACA,WAAWpB,UACPwB,sBACAC,gBAAgB,MAAM,CAACL,UAAU;gBAC/B,gBAAgB,IAAI,CAAC,IAAI,CAAC,cAAc;gBACxC,sBAAsB,IAAI,CAAC,IAAI,CAAC,oBAAoB;gBACpD,cAAc,IAAI,CAAC,IAAI,CAAC,YAAY;gBACpC,oBAAoB,IAAI,CAAC,IAAI,CAAC,kBAAkB;gBAChDC;YACF;YACJ,wBAAwB,IAAIK;YAC5B,aAAaC;YACb,aAAa,EAAE;YACf,kBAAkB,IAAIC;QACxB;QACA,IAAI5B,SAAS;YACXuB,MAAM,SAAS,GAAGvB,QAAQ,cAAc,CAAC,IAAM,IAAI,CAAC,YAAY,CAACuB;YACjEA,MAAM,OAAO,GAAGvB;QAClB;QACA,OAAOuB;IACT;IAEQ,aAAaxB,MAAwB,EAAW;QACtD,OAAOA,OAAO,gBAAgB,CAAC,IAAI,GAAG,KAAK,IAAI,CAAC,OAAO,CAAC,gBAAgB;IAC1E;IAEQ,kBAAkBA,SAAS,IAAI,CAAC,WAAW,EAAQ;QACzDA,OAAO,IAAI,GAAG,IAAI,CAAC,eAAe;QAClCA,OAAO,sBAAsB,GAAG,IAAI2B;QACpC3B,OAAO,aAAa,GAAGK;QACvBL,OAAO,WAAW,CAAC,MAAM,GAAG;IAC9B;IAEQ,mBAAyB;QAC/B8B,OACE,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,IAAI,IACrC,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,IAChC;IAEJ;IAEA,uBAAyC;QACvC,MAAM9B,SAAS,IAAI,CAAC,WAAW;QAC/B8B,OACE,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,IACzB;QAEFA,OACE,CAAC,IAAI,CAAC,WAAW,EACjB;QAEFA,OACE,CAAC9B,OAAO,YAAY,EACpB;QAEF,OAAOA;IACT;IAEA,MAAM,oBAAuB+B,SAA2B,EAAc;QACpE,MAAM/B,SAAS,IAAI,CAAC,oBAAoB;QACxC,MAAMgC,UAAUC,QAAQ,OAAO,GAAG,IAAI,CAACF;QACvC/B,OAAO,gBAAgB,CAAC,GAAG,CAACgC;QAC5B,IAAI;YACF,OAAO,MAAMA;QACf,SAAU;YACRhC,OAAO,gBAAgB,CAAC,MAAM,CAACgC;QACjC;IACF;IAOA,MAAM,YAAuC;QAC3CF,OACE,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,IACzB;QAEF,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,IAAI,CAAC,WAAW;QAC7C,IAAI,CAAC,gBAAgB;QACrB,IAAI,CAAC,WAAW,GAAI;YAClB,IAAI;gBACF,MAAM,IAAI,CAAC,cAAc;YAC3B,EAAE,OAAOjB,OAAO;gBAEdjB,KACE,CAAC,oDAAoD,EAAEsC,OAAOrB,QAAQ;YAE1E;YACAiB,OACE,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,IACzB;YAEF,MAAMT,WAAWc,kBACf,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa,MAAM;YAEtD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAACd;YAC1C,OAAO,IAAI,CAAC,IAAI;QAClB;QACA,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,WAAW;QAC/B,SAAU;YACR,IAAI,CAAC,WAAW,GAAGhB;QACrB;IACF;IAOA,MAAM,iBAA8C;QAClD,MAAML,SAAS,IAAI,CAAC,WAAW;QAC/B,IAAI,CAACA,OAAO,YAAY,EAAE;YACxB,IAAI,CAAC,gBAAgB;YACrBA,OAAO,YAAY,GAAI;gBACrB,MAAMA,OAAO,SAAS,CAAC,KAAK;gBAC5B,MAAMY,OAAO,MAAMZ,OAAO,SAAS,CAAC,QAAQ;gBAC5CA,OAAO,IAAI,GAAGY;gBACd,IAAI,CAAC,iBAAiB,CAACZ;gBACvB,OAAOY;YACT;QACF;QACA,OAAOZ,OAAO,YAAY;IAC5B;IAEA,oBAAoBoC,SAAwB,EAAEC,MAAmB,EAAE;QACjEP,OACE,CAAC,IAAI,CAAC,WAAW,CAAC,YAAY,EAC9B;QAEF,MAAMQ,cAAc,IAAI,CAAC,IAAI;QAC7B,IAAID,QAAQ;YACV,MAAM5B,gBAAgB,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAAC,GAAG,CAAC4B;YAClE,IAAI5B,AAAkBJ,WAAlBI,eAA6B;gBAC/B6B,YAAY,UAAU,CAAC7B,cAAc,GAAG2B;gBACxC;YACF;YACAE,YAAY,UAAU,CAAC,IAAI,CAACF;YAC5B,IAAI,CAAC,WAAW,CAAC,sBAAsB,CAAC,GAAG,CACzCC,QACAC,YAAY,UAAU,CAAC,MAAM,GAAG;YAElC;QACF;QACAA,YAAY,UAAU,CAAC,IAAI,CAACF;IAC9B;IAEA,eAAeG,GAAqC,EAAE;QAEpD,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB;QAEvD,IAAIC,eAAeD,KAAK,mBACtB,OAAO,IAAI,CAAC,IAAI,CAAC,8BAA8B;QAEjD,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS;IAC5B;IAEA,iBAAiBE,aAA2B,EAAE;QAI5C,IAAIC,iBACF,OAAO;QAIT,MAAMC,aAAaF;QACnB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,EACtC,OAAOG,kBAAkBD;QAI3B,MAAME,WAAW,IAAI,CAAC,WAAW,CAAC,WAAW;QAC7C,MAAMC,aAAaC,4BACjB,IAAI,CAAC,oBAAoB,IACzB,IAAI,CAAC,IAAI,EACT;YACEF;YACA,SAAS,IAAI,CAAC,WAAW,CAAC,WAAW;YACrC,YAAY,IAAI,CAAC,UAAU,IAAI;QACjC;QAEF,OAAO,GAAGD,kBAAkB;YAC1BD;YACA,YAAY;gBAAE,iBAAiBE;gBAAU,kBAAkBA;YAAS;QACtE,GAAG,EAAE,EAAEG,+BAA+BF,aAAa;IACrD;IAEA,oBAAoBG,aAA6B,EAAE;QACjD,MAAMjD,SAAS,IAAI,CAAC,WAAW;QAC/B,IAAIA,OAAO,YAAY,EAAE;YACvB,IAAIiD,eACF,MAAM,IAAIC,cACR,iEACA;gBACE,MAAM;YACR;YAGJ;QACF;QACA,MAAMC,OAAOF,iBAAiBjD,OAAO,aAAa;QAClD,IAAImD,MAAM;YACRnD,OAAO,aAAa,GAAGmD;YACvBnD,OAAO,SAAS,CAAC,iBAAiB,CAChCmD,MACA,IAAI,CAAC,aAAa,IAClB,IAAI,CAAC,IAAI,CAAC,gBAAgB;QAE9B;QACAnD,OAAO,IAAI,GAAGA,OAAO,SAAS,CAAC,aAAa;IAC9C;IAGA,MAAM,cAA2C;QAC/C,IAAI,IAAI,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,WAAW;QAC5C,MAAMA,SAAS,IAAI,CAAC,WAAW;QAC/B,IAAIA,OAAO,YAAY,EAAE,OAAOA,OAAO,YAAY;QACnD,IAAI,CAAC,mBAAmB;QACxB,MAAMA,OAAO,SAAS,CAAC,KAAK;QAC5B,OAAOA,OAAO,aAAa,GAAIA,OAAO,IAAI,IAAIK,SAAaA;IAC7D;IAEQ,gBAA4B;QAClC,OAAO;YACL,WAAW,IAAI,CAAC,IAAI,CAAC,SAAS;YAC9B,kBAAkB,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAC5C,YAAY,IAAI,CAAC,IAAI,CAAC,UAAU;YAChC,aAAa,IAAI,CAAC,IAAI,CAAC,WAAW;YAClC,YAAY,IAAI,CAAC,IAAI,CAAC,UAAU;QAClC;IACF;IAEA,IAAI,eAEU;QACZ,OAAO,IAAI,CAAC,mBAAmB,CAAC,EAAE;IACpC;IAEA,IAAI,aAAa+C,QAEJ,EAAE;QAEb,IAAI,CAAC,mBAAmB,GAAG,EAAE;QAE7B,IAAIA,UACF,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAACA;IAElC;IAEA,sBACEC,QAA+D,EACnD;QACZ,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAACA;QAG9B,OAAO;YACL,IAAI,CAAC,wBAAwB,CAACA;QAChC;IACF;IAMA,yBACEA,QAA+D,EACzD;QACN,MAAMC,QAAQ,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAACD;QAC/C,IAAIC,QAAQ,IACV,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAACA,OAAO;IAE3C;IAKA,2BAAiC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,EAAE;IAC/B;IAEA,0BAA0BL,aAA6B,EAAE;QACvD,IAAI,AAAoC,MAApC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAQ;QAC3C,MAAMN,aAAa,IAAI,CAAC,cAAc;QACtC,KAAK,MAAMU,YAAY,IAAI,CAAC,mBAAmB,CAC7C,IAAI;YACFA,SAASV,YAAYM;QACvB,EAAE,OAAOpC,OAAO;YACd0C,QAAQ,KAAK,CAAC,kCAAkC1C;QAClD;IAEJ;IAEA,WAAWwB,MAAkB,EAAQ;QACnC,IAAI,CAAC,cAAc,CAAC,GAAG,CAACA,QAAQ,IAAI,CAAC,WAAW;IAClD;IAEA,oBAAoBA,MAAkB,EAAQ;QAC5C,MAAMrC,SAAS,IAAI,CAAC,cAAc,CAAC,GAAG,CAACqC;QACvCP,OACE9B,WAAW,IAAI,CAAC,WAAW,IAAI,CAACA,OAAO,YAAY,EACnD;IAEJ;IAEA,MAAM,qBAAoC;QACxC,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,KAAK;IACxC;IAEA,MAAM,oBAAmC;QACvC,MAAO,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAC3C,MAAMiC,QAAQ,UAAU,CAAC;eAAI,IAAI,CAAC,WAAW,CAAC,gBAAgB;SAAC;IAEnE;IAEA,MAAM,eAA8B;QAClC,IAAI,IAAI,CAAC,WAAW,EAAE,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,KAAO;IAC5D;IAtdA,YAA6BuB,OAA2B,CAAE;;QAP1D,uBAAQ,eAAR;QACA,uBAAQ,eAAR;QACA,uBAAiB,kBAAjB;QACA,uBAAQ,uBAAR;aAI6BA,OAAO,GAAPA;aALZ,cAAc,GAAG,IAAI7B;aAC9B,mBAAmB,GAEvB,EAAE;QAGJ,MAAM8B,OAAO,IAAI,CAAC,IAAI;QACtB,MAAMpC,WACJoC,KAAK,cAAc,IACnBtB,kBAAkBsB,KAAK,MAAM,IAAID,QAAQ,aAAa,MAAM;QAC9D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,iBAAiB,CACvCnC,UACAoC,KAAK,gBAAgB,EAAE,CAAC,gBAAgB,KAAKpC;IAEjD;AA8cF"}

package/dist/es/agent/utils.mjs

@@ -153,7 +153,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.17.6";
+const getMidsceneVersion = ()=>"1.17.7";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/model-adapter/locate-prompt-order.mjs

@@ -0,0 +1,80 @@
+const parts = [
+    'instructions',
+    'screenshot',
+    'task',
+    'references'
+];
+function parseLocatePromptOrder(order = 'system(instructions):userTogether(screenshot:task):userEach(references)') {
+    const groups = order.split(/(?<=\)):/);
+    const remaining = new Set(parts);
+    function invalid(reason) {
+        return new Error(`Invalid locate promptOrder "${order}": ${reason}`);
+    }
+    const promptGroups = groups.map((group)=>{
+        const match = /^(system|userTogether|userEach)\(([^()]+)\)$/.exec(group);
+        if (!match) throw invalid(`expected system(...), userTogether(...) or userEach(...), got "${group}"`);
+        const groupParts = match[2].split(':');
+        for (const part of groupParts){
+            if (!remaining.delete(part)) throw invalid(`unknown or repeated part "${part}"`);
+            if ('system' === match[1] && ('screenshot' === part || 'references' === part)) throw invalid(`system cannot contain image part "${part}"`);
+        }
+        if ('system' === match[1]) return {
+            mode: 'system',
+            parts: groupParts
+        };
+        return {
+            mode: match[1],
+            parts: groupParts
+        };
+    });
+    if (remaining.size) throw invalid(`missing ${[
+        ...remaining
+    ].join(', ')}`);
+    return {
+        groups: promptGroups
+    };
+}
+function buildLocateMessages(order, input) {
+    const messages = [];
+    for (const group of order.groups){
+        if ('system' === group.mode) {
+            messages.push({
+                role: 'system',
+                content: group.parts.map((part)=>input[part]).join('\n\n')
+            });
+            continue;
+        }
+        const expanded = group.parts.flatMap((part)=>{
+            if ('references' === part) return input.references;
+            const content = 'screenshot' === part ? [
+                {
+                    type: 'image',
+                    url: input.screenshot
+                }
+            ] : input[part];
+            return [
+                {
+                    role: 'user',
+                    content
+                }
+            ];
+        });
+        if ('userEach' === group.mode) messages.push(...expanded);
+        else if (expanded.length) {
+            const content = expanded.flatMap((message)=>'string' == typeof message.content ? [
+                    {
+                        type: 'text',
+                        text: message.content
+                    }
+                ] : message.content);
+            messages.push({
+                role: 'user',
+                content
+            });
+        }
+    }
+    return messages;
+}
+export { buildLocateMessages, parseLocatePromptOrder };
+
+//# sourceMappingURL=locate-prompt-order.mjs.map

package/dist/es/ai-model/model-adapter/locate-prompt-order.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/locate-prompt-order.mjs","sources":["../../../../src/ai-model/model-adapter/locate-prompt-order.ts"],"sourcesContent":["import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n  MessageContent,\n} from '../service-caller/types';\n\nconst parts = ['instructions', 'screenshot', 'task', 'references'] as const;\ntype Part = (typeof parts)[number];\n\nexport type LocateMessages = (\n  | ConversationSystemMessage\n  | ConversationUserMessage\n)[];\n\ntype TextPart = Extract<Part, 'instructions' | 'task'>;\ntype PromptGroup =\n  | { mode: 'system'; parts: TextPart[] }\n  | { mode: 'userTogether' | 'userEach'; parts: Part[] };\n\nexport interface LocatePromptOrder {\n  groups: readonly PromptGroup[];\n}\n\nexport function parseLocatePromptOrder(\n  order = 'system(instructions):userTogether(screenshot:task):userEach(references)',\n): LocatePromptOrder {\n  const groups = order.split(/(?<=\\)):/);\n  const remaining = new Set<string>(parts);\n\n  function invalid(reason: string): Error {\n    return new Error(`Invalid locate promptOrder \"${order}\": ${reason}`);\n  }\n\n  const promptGroups = groups.map((group): PromptGroup => {\n    const match = /^(system|userTogether|userEach)\\(([^()]+)\\)$/.exec(group);\n    if (!match) {\n      throw invalid(\n        `expected system(...), userTogether(...) or userEach(...), got \"${group}\"`,\n      );\n    }\n    const groupParts = match[2].split(':');\n    for (const part of groupParts) {\n      if (!remaining.delete(part)) {\n        throw invalid(`unknown or repeated part \"${part}\"`);\n      }\n      if (\n        match[1] === 'system' &&\n        (part === 'screenshot' || part === 'references')\n      ) {\n        throw invalid(`system cannot contain image part \"${part}\"`);\n      }\n    }\n    if (match[1] === 'system') {\n      return { mode: 'system', parts: groupParts as TextPart[] };\n    }\n    return {\n      mode: match[1] as 'userTogether' | 'userEach',\n      parts: groupParts as Part[],\n    };\n  });\n  if (remaining.size) {\n    throw invalid(`missing ${[...remaining].join(', ')}`);\n  }\n  return { groups: promptGroups };\n}\n\nexport function buildLocateMessages(\n  order: LocatePromptOrder,\n  input: {\n    instructions: string;\n    screenshot: string;\n    task: string;\n    references: ConversationUserMessage[];\n  },\n): LocateMessages {\n  const messages: LocateMessages = [];\n  for (const group of order.groups) {\n    if (group.mode === 'system') {\n      messages.push({\n        role: 'system',\n        content: group.parts.map((part) => input[part]).join('\\n\\n'),\n      });\n      continue;\n    }\n    const expanded = group.parts.flatMap((part): ConversationUserMessage[] => {\n      if (part === 'references') {\n        return input.references;\n      }\n      const content =\n        part === 'screenshot'\n          ? [{ type: 'image' as const, url: input.screenshot }]\n          : input[part];\n      return [{ role: 'user', content }];\n    });\n    if (group.mode === 'userEach') {\n      messages.push(...expanded);\n    } else if (expanded.length) {\n      const content = expanded.flatMap((message): MessageContent[] =>\n        typeof message.content === 'string'\n          ? [{ type: 'text', text: message.content }]\n          : message.content,\n      );\n      messages.push({ role: 'user', content });\n    }\n  }\n  return messages;\n}\n"],"names":["parts","parseLocatePromptOrder","order","groups","remaining","Set","invalid","reason","Error","promptGroups","group","match","groupParts","part","buildLocateMessages","input","messages","expanded","content","message"],"mappings":"AAMA,MAAMA,QAAQ;IAAC;IAAgB;IAAc;IAAQ;CAAa;AAiB3D,SAASC,uBACdC,QAAQ,yEAAyE;IAEjF,MAAMC,SAASD,MAAM,KAAK,CAAC;IAC3B,MAAME,YAAY,IAAIC,IAAYL;IAElC,SAASM,QAAQC,MAAc;QAC7B,OAAO,IAAIC,MAAM,CAAC,4BAA4B,EAAEN,MAAM,GAAG,EAAEK,QAAQ;IACrE;IAEA,MAAME,eAAeN,OAAO,GAAG,CAAC,CAACO;QAC/B,MAAMC,QAAQ,+CAA+C,IAAI,CAACD;QAClE,IAAI,CAACC,OACH,MAAML,QACJ,CAAC,+DAA+D,EAAEI,MAAM,CAAC,CAAC;QAG9E,MAAME,aAAaD,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC;QAClC,KAAK,MAAME,QAAQD,WAAY;YAC7B,IAAI,CAACR,UAAU,MAAM,CAACS,OACpB,MAAMP,QAAQ,CAAC,0BAA0B,EAAEO,KAAK,CAAC,CAAC;YAEpD,IACEF,AAAa,aAAbA,KAAK,CAAC,EAAE,IACPE,CAAAA,AAAS,iBAATA,QAAyBA,AAAS,iBAATA,IAAoB,GAE9C,MAAMP,QAAQ,CAAC,kCAAkC,EAAEO,KAAK,CAAC,CAAC;QAE9D;QACA,IAAIF,AAAa,aAAbA,KAAK,CAAC,EAAE,EACV,OAAO;YAAE,MAAM;YAAU,OAAOC;QAAyB;QAE3D,OAAO;YACL,MAAMD,KAAK,CAAC,EAAE;YACd,OAAOC;QACT;IACF;IACA,IAAIR,UAAU,IAAI,EAChB,MAAME,QAAQ,CAAC,QAAQ,EAAE;WAAIF;KAAU,CAAC,IAAI,CAAC,OAAO;IAEtD,OAAO;QAAE,QAAQK;IAAa;AAChC;AAEO,SAASK,oBACdZ,KAAwB,EACxBa,KAKC;IAED,MAAMC,WAA2B,EAAE;IACnC,KAAK,MAAMN,SAASR,MAAM,MAAM,CAAE;QAChC,IAAIQ,AAAe,aAAfA,MAAM,IAAI,EAAe;YAC3BM,SAAS,IAAI,CAAC;gBACZ,MAAM;gBACN,SAASN,MAAM,KAAK,CAAC,GAAG,CAAC,CAACG,OAASE,KAAK,CAACF,KAAK,EAAE,IAAI,CAAC;YACvD;YACA;QACF;QACA,MAAMI,WAAWP,MAAM,KAAK,CAAC,OAAO,CAAC,CAACG;YACpC,IAAIA,AAAS,iBAATA,MACF,OAAOE,MAAM,UAAU;YAEzB,MAAMG,UACJL,AAAS,iBAATA,OACI;gBAAC;oBAAE,MAAM;oBAAkB,KAAKE,MAAM,UAAU;gBAAC;aAAE,GACnDA,KAAK,CAACF,KAAK;YACjB,OAAO;gBAAC;oBAAE,MAAM;oBAAQK;gBAAQ;aAAE;QACpC;QACA,IAAIR,AAAe,eAAfA,MAAM,IAAI,EACZM,SAAS,IAAI,IAAIC;aACZ,IAAIA,SAAS,MAAM,EAAE;YAC1B,MAAMC,UAAUD,SAAS,OAAO,CAAC,CAACE,UAChC,AAA2B,YAA3B,OAAOA,QAAQ,OAAO,GAClB;oBAAC;wBAAE,MAAM;wBAAQ,MAAMA,QAAQ,OAAO;oBAAC;iBAAE,GACzCA,QAAQ,OAAO;YAErBH,SAAS,IAAI,CAAC;gBAAE,MAAM;gBAAQE;YAAQ;QACxC;IACF;IACA,OAAOF;AACT"}

package/dist/es/ai-model/model-adapter/locate.mjs

@@ -1,6 +1,7 @@
 import { createLocateResultCodec } from "../shared/model-locate-result/factory.mjs";
 import { resolvePlanningTapLocator } from "../workflows/grounding/planning-action-locate.mjs";
 import { createDefaultElementProtocol, createDefaultSearchAreaProtocol } from "./default-locate-protocol.mjs";
+import { parseLocatePromptOrder } from "./locate-prompt-order.mjs";
 const defaultLocateResultFormatDefinition = {
     coordinates: {
         shape: 'bbox',
@@ -31,7 +32,7 @@ function resolveLocate(locate, resolvedCustomPlanner, protocolContext) {
     const searchAreaResultCodec = searchAreaResultFormat ? createLocateResultCodec(searchAreaResultFormat) : void 0;
     return {
         kind: 'standard',
-        userMessageContentOrder: locate?.userMessageContentOrder ?? 'image-first',
+        promptOrder: parseLocatePromptOrder(locate?.promptOrder),
         element: {
             protocol: elementProtocol,
             resultCodec: elementResultCodec

package/dist/es/ai-model/model-adapter/locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/locate.mjs","sources":["../../../../src/ai-model/model-adapter/locate.ts"],"sourcesContent":["import { createLocateResultCodec } from '../shared/model-locate-result/factory';\nimport type { LocateResultFormatDefinition } from '../shared/model-locate-result/types';\nimport { resolvePlanningTapLocator } from '../workflows/grounding/planning-action-locate';\nimport type { ResolvedCustomPlanningDefinition } from './custom-planning-types';\nimport {\n  createDefaultElementProtocol,\n  createDefaultSearchAreaProtocol,\n} from './default-locate-protocol';\nimport type {\n  StandardLocateProtocolContext,\n  StandardLocateProtocolDefinition,\n} from './locate-protocol';\nimport type { LocateAdapter, ModelAdapterDefinition } from './types';\n\nconst defaultLocateResultFormatDefinition: LocateResultFormatDefinition = {\n  coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n};\n\nconst resolveLocateProtocolDefinition = (\n  protocolDefinition: StandardLocateProtocolDefinition,\n  protocolContext: StandardLocateProtocolContext,\n) =>\n  typeof protocolDefinition === 'function'\n    ? protocolDefinition(protocolContext)\n    : protocolDefinition;\n\nexport function resolveLocate(\n  locate: ModelAdapterDefinition['locate'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n  protocolContext: StandardLocateProtocolContext,\n): LocateAdapter {\n  if (locate?.kind === 'custom') {\n    let locateFn = locate.locateFn;\n\n    if (!locateFn) {\n      const planningTapLocator = locate.planningTapLocator;\n\n      if (!planningTapLocator) {\n        throw new Error(\n          'Custom locate definition requires either locateFn or planningTapLocator',\n        );\n      }\n\n      if (!resolvedCustomPlanner) {\n        throw new Error(\n          'Custom planning tap locator requires a custom planning planner definition',\n        );\n      }\n      locateFn = resolvePlanningTapLocator(\n        planningTapLocator,\n        resolvedCustomPlanner,\n      );\n    }\n\n    return {\n      kind: 'custom',\n      locateFn,\n    };\n  }\n\n  const elementProtocol = resolveLocateProtocolDefinition(\n    locate?.element?.protocol ?? createDefaultElementProtocol,\n    protocolContext,\n  );\n  const elementResultFormat =\n    locate?.element?.resultFormat ?? defaultLocateResultFormatDefinition;\n  const elementResultCodec = createLocateResultCodec(elementResultFormat);\n  const searchAreaProtocol =\n    locate?.searchArea === false\n      ? undefined\n      : resolveLocateProtocolDefinition(\n          locate?.searchArea?.protocol ?? createDefaultSearchAreaProtocol,\n          protocolContext,\n        );\n  const searchAreaResultFormat =\n    locate?.searchArea === false\n      ? undefined\n      : (locate?.searchArea?.resultFormat ?? elementResultFormat);\n  const searchAreaResultCodec = searchAreaResultFormat\n    ? createLocateResultCodec(searchAreaResultFormat)\n    : undefined;\n\n  return {\n    kind: 'standard',\n    userMessageContentOrder: locate?.userMessageContentOrder ?? 'image-first',\n    element: {\n      protocol: elementProtocol,\n      resultCodec: elementResultCodec,\n    },\n    ...(searchAreaProtocol && searchAreaResultCodec\n      ? {\n          searchArea: {\n            protocol: searchAreaProtocol,\n            resultCodec: searchAreaResultCodec,\n          },\n        }\n      : {}),\n  };\n}\n"],"names":["defaultLocateResultFormatDefinition","resolveLocateProtocolDefinition","protocolDefinition","protocolContext","resolveLocate","locate","resolvedCustomPlanner","locateFn","planningTapLocator","Error","resolvePlanningTapLocator","elementProtocol","createDefaultElementProtocol","elementResultFormat","elementResultCodec","createLocateResultCodec","searchAreaProtocol","undefined","createDefaultSearchAreaProtocol","searchAreaResultFormat","searchAreaResultCodec"],"mappings":";;;AAcA,MAAMA,sCAAoE;IACxE,aAAa;QAAE,OAAO;QAAQ,OAAO;QAAM,cAAc;IAAK;AAChE;AAEA,MAAMC,kCAAkC,CACtCC,oBACAC,kBAEA,AAA8B,cAA9B,OAAOD,qBACHA,mBAAmBC,mBACnBD;AAEC,SAASE,cACdC,MAAwC,EACxCC,qBAAmE,EACnEH,eAA8C;IAE9C,IAAIE,QAAQ,SAAS,UAAU;QAC7B,IAAIE,WAAWF,OAAO,QAAQ;QAE9B,IAAI,CAACE,UAAU;YACb,MAAMC,qBAAqBH,OAAO,kBAAkB;YAEpD,IAAI,CAACG,oBACH,MAAM,IAAIC,MACR;YAIJ,IAAI,CAACH,uBACH,MAAM,IAAIG,MACR;YAGJF,WAAWG,0BACTF,oBACAF;QAEJ;QAEA,OAAO;YACL,MAAM;YACNC;QACF;IACF;IAEA,MAAMI,kBAAkBV,gCACtBI,QAAQ,SAAS,YAAYO,8BAC7BT;IAEF,MAAMU,sBACJR,QAAQ,SAAS,gBAAgBL;IACnC,MAAMc,qBAAqBC,wBAAwBF;IACnD,MAAMG,qBACJX,QAAQ,eAAe,QACnBY,SACAhB,gCACEI,QAAQ,YAAY,YAAYa,iCAChCf;IAER,MAAMgB,yBACJd,QAAQ,eAAe,QACnBY,SACCZ,QAAQ,YAAY,gBAAgBQ;IAC3C,MAAMO,wBAAwBD,yBAC1BJ,wBAAwBI,0BACxBF;IAEJ,OAAO;QACL,MAAM;QACN,yBAAyBZ,QAAQ,2BAA2B;QAC5D,SAAS;YACP,UAAUM;YACV,aAAaG;QACf;QACA,GAAIE,sBAAsBI,wBACtB;YACE,YAAY;gBACV,UAAUJ;gBACV,aAAaI;YACf;QACF,IACA,CAAC,CAAC;IACR;AACF"}
+{"version":3,"file":"ai-model/model-adapter/locate.mjs","sources":["../../../../src/ai-model/model-adapter/locate.ts"],"sourcesContent":["import { createLocateResultCodec } from '../shared/model-locate-result/factory';\nimport type { LocateResultFormatDefinition } from '../shared/model-locate-result/types';\nimport { resolvePlanningTapLocator } from '../workflows/grounding/planning-action-locate';\nimport type { ResolvedCustomPlanningDefinition } from './custom-planning-types';\nimport {\n  createDefaultElementProtocol,\n  createDefaultSearchAreaProtocol,\n} from './default-locate-protocol';\nimport { parseLocatePromptOrder } from './locate-prompt-order';\nimport type {\n  StandardLocateProtocolContext,\n  StandardLocateProtocolDefinition,\n} from './locate-protocol';\nimport type { LocateAdapter, ModelAdapterDefinition } from './types';\n\nconst defaultLocateResultFormatDefinition: LocateResultFormatDefinition = {\n  coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n};\n\nconst resolveLocateProtocolDefinition = (\n  protocolDefinition: StandardLocateProtocolDefinition,\n  protocolContext: StandardLocateProtocolContext,\n) =>\n  typeof protocolDefinition === 'function'\n    ? protocolDefinition(protocolContext)\n    : protocolDefinition;\n\nexport function resolveLocate(\n  locate: ModelAdapterDefinition['locate'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n  protocolContext: StandardLocateProtocolContext,\n): LocateAdapter {\n  if (locate?.kind === 'custom') {\n    let locateFn = locate.locateFn;\n\n    if (!locateFn) {\n      const planningTapLocator = locate.planningTapLocator;\n\n      if (!planningTapLocator) {\n        throw new Error(\n          'Custom locate definition requires either locateFn or planningTapLocator',\n        );\n      }\n\n      if (!resolvedCustomPlanner) {\n        throw new Error(\n          'Custom planning tap locator requires a custom planning planner definition',\n        );\n      }\n      locateFn = resolvePlanningTapLocator(\n        planningTapLocator,\n        resolvedCustomPlanner,\n      );\n    }\n\n    return {\n      kind: 'custom',\n      locateFn,\n    };\n  }\n\n  const elementProtocol = resolveLocateProtocolDefinition(\n    locate?.element?.protocol ?? createDefaultElementProtocol,\n    protocolContext,\n  );\n  const elementResultFormat =\n    locate?.element?.resultFormat ?? defaultLocateResultFormatDefinition;\n  const elementResultCodec = createLocateResultCodec(elementResultFormat);\n  const searchAreaProtocol =\n    locate?.searchArea === false\n      ? undefined\n      : resolveLocateProtocolDefinition(\n          locate?.searchArea?.protocol ?? createDefaultSearchAreaProtocol,\n          protocolContext,\n        );\n  const searchAreaResultFormat =\n    locate?.searchArea === false\n      ? undefined\n      : (locate?.searchArea?.resultFormat ?? elementResultFormat);\n  const searchAreaResultCodec = searchAreaResultFormat\n    ? createLocateResultCodec(searchAreaResultFormat)\n    : undefined;\n\n  return {\n    kind: 'standard',\n    promptOrder: parseLocatePromptOrder(locate?.promptOrder),\n    element: {\n      protocol: elementProtocol,\n      resultCodec: elementResultCodec,\n    },\n    ...(searchAreaProtocol && searchAreaResultCodec\n      ? {\n          searchArea: {\n            protocol: searchAreaProtocol,\n            resultCodec: searchAreaResultCodec,\n          },\n        }\n      : {}),\n  };\n}\n"],"names":["defaultLocateResultFormatDefinition","resolveLocateProtocolDefinition","protocolDefinition","protocolContext","resolveLocate","locate","resolvedCustomPlanner","locateFn","planningTapLocator","Error","resolvePlanningTapLocator","elementProtocol","createDefaultElementProtocol","elementResultFormat","elementResultCodec","createLocateResultCodec","searchAreaProtocol","undefined","createDefaultSearchAreaProtocol","searchAreaResultFormat","searchAreaResultCodec","parseLocatePromptOrder"],"mappings":";;;;AAeA,MAAMA,sCAAoE;IACxE,aAAa;QAAE,OAAO;QAAQ,OAAO;QAAM,cAAc;IAAK;AAChE;AAEA,MAAMC,kCAAkC,CACtCC,oBACAC,kBAEA,AAA8B,cAA9B,OAAOD,qBACHA,mBAAmBC,mBACnBD;AAEC,SAASE,cACdC,MAAwC,EACxCC,qBAAmE,EACnEH,eAA8C;IAE9C,IAAIE,QAAQ,SAAS,UAAU;QAC7B,IAAIE,WAAWF,OAAO,QAAQ;QAE9B,IAAI,CAACE,UAAU;YACb,MAAMC,qBAAqBH,OAAO,kBAAkB;YAEpD,IAAI,CAACG,oBACH,MAAM,IAAIC,MACR;YAIJ,IAAI,CAACH,uBACH,MAAM,IAAIG,MACR;YAGJF,WAAWG,0BACTF,oBACAF;QAEJ;QAEA,OAAO;YACL,MAAM;YACNC;QACF;IACF;IAEA,MAAMI,kBAAkBV,gCACtBI,QAAQ,SAAS,YAAYO,8BAC7BT;IAEF,MAAMU,sBACJR,QAAQ,SAAS,gBAAgBL;IACnC,MAAMc,qBAAqBC,wBAAwBF;IACnD,MAAMG,qBACJX,QAAQ,eAAe,QACnBY,SACAhB,gCACEI,QAAQ,YAAY,YAAYa,iCAChCf;IAER,MAAMgB,yBACJd,QAAQ,eAAe,QACnBY,SACCZ,QAAQ,YAAY,gBAAgBQ;IAC3C,MAAMO,wBAAwBD,yBAC1BJ,wBAAwBI,0BACxBF;IAEJ,OAAO;QACL,MAAM;QACN,aAAaI,uBAAuBhB,QAAQ;QAC5C,SAAS;YACP,UAAUM;YACV,aAAaG;QACf;QACA,GAAIE,sBAAsBI,wBACtB;YACE,YAAY;gBACV,UAAUJ;gBACV,aAAaI;YACf;QACF,IACA,CAAC,CAAC;IACR;AACF"}

package/dist/es/ai-model/workflows/grounding/locate.mjs

@@ -84,7 +84,7 @@ async function genericLocate(locateRequest) {
         locateImage: locateRequest.locateImage,
         imagePreprocess: adapter.imagePreprocess,
         targetDescription: targetElementDescription,
-        userMessageContentOrder: adapter.locate.userMessageContentOrder
+        promptOrder: adapter.locate.promptOrder
     });
     try {
         return await callAiAndParseWithRetry({

package/dist/es/ai-model/workflows/grounding/locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;AAsBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,oCACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,mBAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,OACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,yBAAyByB,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,yBAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,qBACTD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    promptOrder: adapter.locate.promptOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;AAsBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,oCACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,mBAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,OACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,aAAayB,QAAQ,MAAM,CAAC,WAAW;IACzC;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,yBAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,qBACTD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/section-locate.mjs

@@ -31,7 +31,7 @@ async function AiLocateSection(options) {
         },
         imagePreprocess: adapter.imagePreprocess,
         targetDescription: sectionDescription,
-        userMessageContentOrder: adapter.locate.userMessageContentOrder
+        promptOrder: adapter.locate.promptOrder
     });
     let parsedResult;
     try {

package/dist/es/ai-model/workflows/grounding/section-locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;AAwBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,uBAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    promptOrder: adapter.locate.promptOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;AAwBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,aAAaE,QAAQ,MAAM,CAAC,WAAW;IACzC;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,uBAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/utils.mjs

@@ -1,7 +1,8 @@
 import { userPromptToMultimodalPrompt } from "../../../common.mjs";
 import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
+import { buildLocateMessages } from "../../model-adapter/locate-prompt-order.mjs";
 import { multimodalPromptToMessages } from "../../shared/multimodal-prompt.mjs";
-async function prepareLocateModelInput({ locateImage, targetDescription, systemPrompt, userPrompt, imagePreprocess, userMessageContentOrder }) {
+async function prepareLocateModelInput({ locateImage, targetDescription, systemPrompt, userPrompt, imagePreprocess, promptOrder }) {
     const preparedImage = await prepareModelImage({
         imageBase64: locateImage.imageBase64,
         width: locateImage.width,
@@ -9,33 +10,14 @@ async function prepareLocateModelInput({ locateImage, targetDescription, systemP
         policy: imagePreprocess
     });
     const referenceImageMessages = await multimodalPromptToMessages(userPromptToMultimodalPrompt(targetDescription));
-    const imageContent = {
-        type: 'image',
-        url: preparedImage.imageBase64
-    };
-    const promptContent = {
-        type: 'text',
-        text: userPrompt
-    };
     return {
         preparedImage,
-        messages: [
-            {
-                role: 'system',
-                content: systemPrompt
-            },
-            {
-                role: 'user',
-                content: 'prompt-first' === userMessageContentOrder ? [
-                    promptContent,
-                    imageContent
-                ] : [
-                    imageContent,
-                    promptContent
-                ]
-            },
-            ...referenceImageMessages
-        ]
+        messages: buildLocateMessages(promptOrder, {
+            instructions: systemPrompt,
+            screenshot: preparedImage.imageBase64,
+            task: userPrompt,
+            references: referenceImageMessages
+        })
     };
 }
 function formatLocateModelContext(modelRuntime) {

package/dist/es/ai-model/workflows/grounding/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/utils.mjs","sources":["../../../../../src/ai-model/workflows/grounding/utils.ts"],"sourcesContent":["import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n  MessageContent,\n} from '@/ai-model/service-caller/types';\nimport { type TUserPrompt, userPromptToMultimodalPrompt } from '@/common';\nimport {\n  type ImagePreprocessPolicy,\n  type PreparedModelImage,\n  prepareModelImage,\n} from '../../model-adapter/image-preprocess';\nimport type { LocateUserMessageContentOrder } from '../../model-adapter/types';\nimport type { ModelRuntime } from '../../models';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\n\nexport type GroundingAIArgs = [\n  ConversationSystemMessage,\n  ...ConversationUserMessage[],\n];\n\nexport async function prepareLocateModelInput({\n  locateImage,\n  targetDescription,\n  systemPrompt,\n  userPrompt,\n  imagePreprocess,\n  userMessageContentOrder,\n}: {\n  locateImage: {\n    imageBase64: string;\n    width: number;\n    height: number;\n  };\n  targetDescription: TUserPrompt;\n  systemPrompt: string;\n  userPrompt: string;\n  imagePreprocess: ImagePreprocessPolicy;\n  userMessageContentOrder: LocateUserMessageContentOrder;\n}): Promise<{\n  messages: GroundingAIArgs;\n  preparedImage: PreparedModelImage;\n}> {\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateImage.imageBase64,\n    width: locateImage.width,\n    height: locateImage.height,\n    policy: imagePreprocess,\n  });\n  const referenceImageMessages = await multimodalPromptToMessages(\n    userPromptToMultimodalPrompt(targetDescription),\n  );\n  const imageContent: MessageContent = {\n    type: 'image',\n    url: preparedImage.imageBase64,\n  };\n  const promptContent: MessageContent = {\n    type: 'text',\n    text: userPrompt,\n  };\n\n  return {\n    preparedImage,\n    messages: [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content:\n          userMessageContentOrder === 'prompt-first'\n            ? [promptContent, imageContent]\n            : [imageContent, promptContent],\n      },\n      ...referenceImageMessages,\n    ],\n  };\n}\n\nexport function formatLocateModelContext(modelRuntime: ModelRuntime): string {\n  const { modelFamily, modelName } = modelRuntime.config;\n  return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;\n}\n"],"names":["prepareLocateModelInput","locateImage","targetDescription","systemPrompt","userPrompt","imagePreprocess","userMessageContentOrder","preparedImage","prepareModelImage","referenceImageMessages","multimodalPromptToMessages","userPromptToMultimodalPrompt","imageContent","promptContent","formatLocateModelContext","modelRuntime","modelFamily","modelName"],"mappings":";;;AAoBO,eAAeA,wBAAwB,EAC5CC,WAAW,EACXC,iBAAiB,EACjBC,YAAY,EACZC,UAAU,EACVC,eAAe,EACfC,uBAAuB,EAYxB;IAIC,MAAMC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,YAAY,WAAW;QACpC,OAAOA,YAAY,KAAK;QACxB,QAAQA,YAAY,MAAM;QAC1B,QAAQI;IACV;IACA,MAAMI,yBAAyB,MAAMC,2BACnCC,6BAA6BT;IAE/B,MAAMU,eAA+B;QACnC,MAAM;QACN,KAAKL,cAAc,WAAW;IAChC;IACA,MAAMM,gBAAgC;QACpC,MAAM;QACN,MAAMT;IACR;IAEA,OAAO;QACLG;QACA,UAAU;YACR;gBAAE,MAAM;gBAAU,SAASJ;YAAa;YACxC;gBACE,MAAM;gBACN,SACEG,AAA4B,mBAA5BA,0BACI;oBAACO;oBAAeD;iBAAa,GAC7B;oBAACA;oBAAcC;iBAAc;YACrC;eACGJ;SACJ;IACH;AACF;AAEO,SAASK,yBAAyBC,YAA0B;IACjE,MAAM,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGF,aAAa,MAAM;IACtD,OAAO,CAAC,UAAU,EAAEE,aAAa,QAAQ,aAAa,EAAED,eAAe,SAAS;AAClF"}
+{"version":3,"file":"ai-model/workflows/grounding/utils.mjs","sources":["../../../../../src/ai-model/workflows/grounding/utils.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToMultimodalPrompt } from '@/common';\nimport {\n  type ImagePreprocessPolicy,\n  type PreparedModelImage,\n  prepareModelImage,\n} from '../../model-adapter/image-preprocess';\nimport {\n  type LocateMessages,\n  type LocatePromptOrder,\n  buildLocateMessages,\n} from '../../model-adapter/locate-prompt-order';\nimport type { ModelRuntime } from '../../models';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\n\nexport async function prepareLocateModelInput({\n  locateImage,\n  targetDescription,\n  systemPrompt,\n  userPrompt,\n  imagePreprocess,\n  promptOrder,\n}: {\n  locateImage: {\n    imageBase64: string;\n    width: number;\n    height: number;\n  };\n  targetDescription: TUserPrompt;\n  systemPrompt: string;\n  userPrompt: string;\n  imagePreprocess: ImagePreprocessPolicy;\n  promptOrder: LocatePromptOrder;\n}): Promise<{\n  messages: LocateMessages;\n  preparedImage: PreparedModelImage;\n}> {\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateImage.imageBase64,\n    width: locateImage.width,\n    height: locateImage.height,\n    policy: imagePreprocess,\n  });\n  const referenceImageMessages = await multimodalPromptToMessages(\n    userPromptToMultimodalPrompt(targetDescription),\n  );\n  return {\n    preparedImage,\n    messages: buildLocateMessages(promptOrder, {\n      instructions: systemPrompt,\n      screenshot: preparedImage.imageBase64,\n      task: userPrompt,\n      references: referenceImageMessages,\n    }),\n  };\n}\n\nexport function formatLocateModelContext(modelRuntime: ModelRuntime): string {\n  const { modelFamily, modelName } = modelRuntime.config;\n  return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;\n}\n"],"names":["prepareLocateModelInput","locateImage","targetDescription","systemPrompt","userPrompt","imagePreprocess","promptOrder","preparedImage","prepareModelImage","referenceImageMessages","multimodalPromptToMessages","userPromptToMultimodalPrompt","buildLocateMessages","formatLocateModelContext","modelRuntime","modelFamily","modelName"],"mappings":";;;;AAcO,eAAeA,wBAAwB,EAC5CC,WAAW,EACXC,iBAAiB,EACjBC,YAAY,EACZC,UAAU,EACVC,eAAe,EACfC,WAAW,EAYZ;IAIC,MAAMC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,YAAY,WAAW;QACpC,OAAOA,YAAY,KAAK;QACxB,QAAQA,YAAY,MAAM;QAC1B,QAAQI;IACV;IACA,MAAMI,yBAAyB,MAAMC,2BACnCC,6BAA6BT;IAE/B,OAAO;QACLK;QACA,UAAUK,oBAAoBN,aAAa;YACzC,cAAcH;YACd,YAAYI,cAAc,WAAW;YACrC,MAAMH;YACN,YAAYK;QACd;IACF;AACF;AAEO,SAASI,yBAAyBC,YAA0B;IACjE,MAAM,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGF,aAAa,MAAM;IACtD,OAAO,CAAC,UAAU,EAAEE,aAAa,QAAQ,aAAa,EAAED,eAAe,SAAS;AAClF"}

package/dist/es/dump/report-image-export.mjs

@@ -0,0 +1,58 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { imageRefFileExtension } from "./image-reference.mjs";
+import { parseBase64ImageDataUrl, reportImageFileName, resolveImageSource } from "./screenshot-store.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class ReportImageExporter {
+    write(ref, dataUri) {
+        const key = JSON.stringify([
+            ref.id,
+            ref.mimeType,
+            ref.storage,
+            ref.path
+        ]);
+        const known = this.locations.get(key);
+        if (known) return known;
+        const source = dataUri ? {
+            type: 'data-uri',
+            dataUri
+        } : resolveImageSource(ref, {
+            reportPath: this.reportPath
+        });
+        const bytes = 'file' === source.type ? readFileSync(source.filePath) : Buffer.from(parseBase64ImageDataUrl(source.dataUri).rawBase64, 'base64');
+        const name = reportImageFileName(imageRefFileExtension(ref), bytes);
+        if (!this.writtenFiles.has(name)) {
+            writeFileSync(join(this.screenshotsDir, name), bytes);
+            this.writtenFiles.add(name);
+        }
+        this.locations.set(key, name);
+        return name;
+    }
+    get files() {
+        return [
+            ...this.writtenFiles
+        ].sort().map((name)=>join(this.screenshotsDir, name));
+    }
+    constructor(reportPath, screenshotsDir){
+        _define_property(this, "reportPath", void 0);
+        _define_property(this, "screenshotsDir", void 0);
+        _define_property(this, "locations", void 0);
+        _define_property(this, "writtenFiles", void 0);
+        this.reportPath = reportPath;
+        this.screenshotsDir = screenshotsDir;
+        this.locations = new Map();
+        this.writtenFiles = new Set();
+    }
+}
+export { ReportImageExporter };
+
+//# sourceMappingURL=report-image-export.mjs.map

package/dist/es/dump/report-image-export.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"dump/report-image-export.mjs","sources":["../../../src/dump/report-image-export.ts"],"sourcesContent":["import { readFileSync, writeFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { type StoredImageRef, imageRefFileExtension } from './image-reference';\nimport {\n  parseBase64ImageDataUrl,\n  reportImageFileName,\n  resolveImageSource,\n} from './screenshot-store';\n\n/** Export image bytes once while keeping every observation's own metadata. */\nexport class ReportImageExporter {\n  private readonly locations = new Map<string, string>();\n  private readonly writtenFiles = new Set<string>();\n\n  constructor(\n    private readonly reportPath: string,\n    private readonly screenshotsDir: string,\n  ) {}\n\n  write(ref: StoredImageRef, dataUri?: string): string {\n    const key = JSON.stringify([ref.id, ref.mimeType, ref.storage, ref.path]);\n    const known = this.locations.get(key);\n    if (known) return known;\n    const source = dataUri\n      ? { type: 'data-uri' as const, dataUri }\n      : resolveImageSource(ref, { reportPath: this.reportPath });\n    const bytes =\n      source.type === 'file'\n        ? readFileSync(source.filePath)\n        : Buffer.from(\n            parseBase64ImageDataUrl(source.dataUri).rawBase64,\n            'base64',\n          );\n    const name = reportImageFileName(imageRefFileExtension(ref), bytes);\n    if (!this.writtenFiles.has(name)) {\n      writeFileSync(join(this.screenshotsDir, name), bytes);\n      this.writtenFiles.add(name);\n    }\n    this.locations.set(key, name);\n    return name;\n  }\n\n  get files(): string[] {\n    return [...this.writtenFiles]\n      .sort()\n      .map((name) => join(this.screenshotsDir, name));\n  }\n}\n"],"names":["ReportImageExporter","ref","dataUri","key","JSON","known","source","resolveImageSource","bytes","readFileSync","Buffer","parseBase64ImageDataUrl","name","reportImageFileName","imageRefFileExtension","writeFileSync","join","reportPath","screenshotsDir","Map","Set"],"mappings":";;;;;;;;;;;;;;AAUO,MAAMA;IASX,MAAMC,GAAmB,EAAEC,OAAgB,EAAU;QACnD,MAAMC,MAAMC,KAAK,SAAS,CAAC;YAACH,IAAI,EAAE;YAAEA,IAAI,QAAQ;YAAEA,IAAI,OAAO;YAAEA,IAAI,IAAI;SAAC;QACxE,MAAMI,QAAQ,IAAI,CAAC,SAAS,CAAC,GAAG,CAACF;QACjC,IAAIE,OAAO,OAAOA;QAClB,MAAMC,SAASJ,UACX;YAAE,MAAM;YAAqBA;QAAQ,IACrCK,mBAAmBN,KAAK;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC;QAC1D,MAAMO,QACJF,AAAgB,WAAhBA,OAAO,IAAI,GACPG,aAAaH,OAAO,QAAQ,IAC5BI,OAAO,IAAI,CACTC,wBAAwBL,OAAO,OAAO,EAAE,SAAS,EACjD;QAER,MAAMM,OAAOC,oBAAoBC,sBAAsBb,MAAMO;QAC7D,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAACI,OAAO;YAChCG,cAAcC,KAAK,IAAI,CAAC,cAAc,EAAEJ,OAAOJ;YAC/C,IAAI,CAAC,YAAY,CAAC,GAAG,CAACI;QACxB;QACA,IAAI,CAAC,SAAS,CAAC,GAAG,CAACT,KAAKS;QACxB,OAAOA;IACT;IAEA,IAAI,QAAkB;QACpB,OAAO;eAAI,IAAI,CAAC,YAAY;SAAC,CAC1B,IAAI,GACJ,GAAG,CAAC,CAACA,OAASI,KAAK,IAAI,CAAC,cAAc,EAAEJ;IAC7C;IAhCA,YACmBK,UAAkB,EAClBC,cAAsB,CACvC;;;QANF,uBAAiB,aAAjB;QACA,uBAAiB,gBAAjB;aAGmBD,UAAU,GAAVA;aACAC,cAAc,GAAdA;aALF,SAAS,GAAG,IAAIC;aAChB,YAAY,GAAG,IAAIC;IAKjC;AA8BL"}

package/dist/es/dump/screenshot-store.mjs

@@ -87,6 +87,11 @@ class ReportImageStore {
         if (!file) throw new Error(`ReportImageStore: image ${id} has no persisted file`);
         return file.relativePath;
     }
+    persistFileCopy(screenshot) {
+        return this.persistToSharedFileIfNeeded(screenshot, {
+            markAsPersisted: false
+        });
+    }
     async persist(screenshot) {
         await this.ensureExistingInlineImageIdsLoaded();
         const shouldWriteFileCopy = 'directory' === this.mode || this.alsoWriteFileCopy;