@midscene/core 1.11.0 → 1.11.1-beta-20260818123028.0

package/dist/es/agent/ui-observer.mjs

@@ -1,5 +1,5 @@
 import { UIObservationRecordWriter, cloneUIObservationRecord } from "@midscene/shared/agent-tools/observation-record";
-import { imageInfoOfBase64, resizeImgBase64 } from "@midscene/shared/img";
+import { convertPngBase64ToJpeg, imageInfoOfBase64, resizeImgBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { ScreenshotItem } from "../screenshot-item.mjs";
@@ -24,6 +24,7 @@ const FIRST_FRAME_TIMEOUT_MS = 3000;
 const DEFAULT_WATCHDOG_MS = 300000;
 const MAX_FRAMES_PER_RECORD = 50;
 const DECODE_BATCH_SIZE = 4;
+const OBSERVATION_JPEG_QUALITY = 90;
 class UIObservationImpl {
     get frameCount() {
         return this.record.frames.length;
@@ -186,7 +187,7 @@ class UIObserverImpl {
         });
         if (!this.representativeFrame) {
             const representative = this.representative;
-            this.representativeFrame = this.writer.persistFrame(representative.screenshot.base64, representative.screenshot.capturedAt);
+            this.representativeFrame = this.writer.persistFrame(await this.prepareFrameForPersistence(representative.screenshot.base64), representative.screenshot.capturedAt);
             representative.screenshot = ScreenshotItem.fromFile(this.writer.resolveFramePath(this.representativeFrame), this.representativeFrame.mimeType, this.representativeFrame.capturedAt);
         }
         const frames = [
@@ -232,10 +233,8 @@ class UIObserverImpl {
                 return frame.ref;
             });
             assert(decoded.length === batch.length, 'frame source decode() must return one image per frame handle');
-            for(let index = 0; index < batch.length; index++){
-                const dataUrl = await this.shrinkIfNeeded(decoded[index]);
-                this.persistedByRef.set(batch[index].ref, this.writer.persistFrame(dataUrl, batch[index].capturedAt));
-            }
+            const preparedDataUrls = await Promise.all(decoded.map((dataUrl)=>this.prepareFrameForPersistence(dataUrl)));
+            for(let index = 0; index < batch.length; index++)this.persistedByRef.set(batch[index].ref, this.writer.persistFrame(preparedDataUrls[index], batch[index].capturedAt));
         }
         for (const frame of this.frames)frame.persisted ??= this.persistedByRef.get(frame.ref);
         if (uncachedFrames.length > 0) debug(`decoded and persisted ${uncachedFrames.length} source frames`);
@@ -246,7 +245,7 @@ class UIObserverImpl {
                 const frame = this.source.latest();
                 if (!frame) return;
                 if (isImageDataUrl(frame.ref)) {
-                    const dataUrl = await this.shrinkIfNeeded(frame.ref);
+                    const dataUrl = await this.prepareFrameForPersistence(frame.ref);
                     const persisted = this.writer.persistFrame(dataUrl, frame.capturedAt);
                     this.pushFrame({
                         ref: persisted.path,
@@ -256,7 +255,7 @@ class UIObserverImpl {
                 } else this.pushFrame(frame);
                 return;
             }
-            const dataUrl = await this.shrinkIfNeeded(await this.deps.screenshot());
+            const dataUrl = await this.prepareFrameForPersistence(await this.deps.screenshot());
             const persisted = this.writer.persistFrame(dataUrl, Date.now());
             this.pushFrame({
                 ref: persisted.path,
@@ -267,13 +266,16 @@ class UIObserverImpl {
             debug(`frame capture failed, skipping tick: ${error}`);
         }
     }
-    async shrinkIfNeeded(dataUrl) {
-        if (this.screenshotShrinkFactor <= 1) return dataUrl;
-        const { width, height } = await imageInfoOfBase64(dataUrl);
-        return resizeImgBase64(dataUrl, {
-            width: Math.round(width / this.screenshotShrinkFactor),
-            height: Math.round(height / this.screenshotShrinkFactor)
-        });
+    async prepareFrameForPersistence(dataUrl) {
+        let preparedDataUrl = dataUrl;
+        if (this.screenshotShrinkFactor > 1) {
+            const { width, height } = await imageInfoOfBase64(dataUrl);
+            preparedDataUrl = await resizeImgBase64(dataUrl, {
+                width: Math.round(width / this.screenshotShrinkFactor),
+                height: Math.round(height / this.screenshotShrinkFactor)
+            });
+        }
+        return convertPngBase64ToJpeg(preparedDataUrl, OBSERVATION_JPEG_QUALITY);
     }
     async runLoop() {
         while(!this.stopped){

package/dist/es/agent/ui-observer.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-observer.mjs","sources":["../../../src/agent/ui-observer.ts"],"sourcesContent":["import {\n  type UIObservationRecordMetadata,\n  UIObservationRecordWriter,\n  cloneUIObservationRecord,\n} from '@midscene/shared/agent-tools/observation-record';\nimport type {\n  BaseUIObserverOptions,\n  UIObservationFrame,\n  UIObservationRecord,\n} from '@midscene/shared/agent-tools/types';\nimport { imageInfoOfBase64, resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../common';\nimport type { DeviceFrameRef, DeviceFrameSource } from '../device';\nimport { ScreenshotItem } from '../screenshot-item';\nimport type {\n  AgentAssertResult,\n  InsightAPI,\n  ObservationAssertOptions,\n  ObservationQueryOptions,\n  ServiceExtractParam,\n  UIContext,\n} from '../types';\n\nconst debug = getDebug('ui-observer');\nconst warnObserver = getDebug('ui-observer', { console: true });\n\nconst DEFAULT_INTERVAL_MS = 1000;\nconst MIN_INTERVAL_MS = 200;\nconst DEFAULT_MAX_FRAMES = 30;\nconst FIRST_FRAME_TIMEOUT_MS = 3000;\nconst DEFAULT_WATCHDOG_MS = 5 * 60 * 1000;\nconst MAX_FRAMES_PER_RECORD = 50;\nconst DECODE_BATCH_SIZE = 4;\n\n/** Options for a UI observation window. */\nexport type UIObserverOption = BaseUIObserverOptions;\n\ninterface UIObserverDeps {\n  openFrameSource: () => Promise<DeviceFrameSource | undefined>;\n  screenshot: () => Promise<string>;\n  captureRepresentative: () => Promise<UIContext>;\n  createInsight: (record: UIObservationRecord) => InsightAPI;\n  onStopped?: () => void;\n  onDisposed?: () => void;\n  screenshotShrinkFactor?: number;\n  /** Test/internal persistence override; not part of the Agent SDK options. */\n  observationRecordWriter?: UIObservationRecordWriter;\n}\n\n/** A fixed screen-recording window that supports read-only AI insights. */\nexport interface UIObservation\n  extends InsightAPI<ObservationQueryOptions, ObservationAssertOptions> {\n  /** Number of captured frames in the fixed observation window. */\n  readonly frameCount: number;\n  /** Timestamp when screen sampling started. */\n  readonly startedAt: number;\n  /** Timestamp when screen sampling ended. */\n  readonly endedAt: number;\n  /** Release image files owned by this observation. Failed cleanup is retryable. */\n  dispose(): Promise<void>;\n}\n\n/** Recording lifecycle returned by {@link Agent.startObserving}. */\nexport interface UIObserver {\n  /** Number of frames currently buffered while recording. */\n  readonly bufferedFrameCount: number;\n  /** Stop recording and return its fixed observation window. */\n  stop(): Promise<UIObservation>;\n  /** Stop recording if needed and release its image files. */\n  dispose(): Promise<void>;\n}\n\n/** @internal Concrete fixed-window implementation. */\nexport class UIObservationImpl implements UIObservation {\n  private disposed = false;\n  private readonly record: UIObservationRecord;\n\n  constructor(\n    record: UIObservationRecord,\n    private readonly insight: InsightAPI,\n    private readonly disposeRecord?: () => void,\n    private readonly onDisposed?: () => void,\n  ) {\n    this.record = cloneUIObservationRecord(record);\n  }\n\n  get frameCount(): number {\n    return this.record.frames.length;\n  }\n\n  get startedAt(): number {\n    return this.record.startedAt;\n  }\n\n  get endedAt(): number {\n    return this.record.endedAt;\n  }\n\n  private ensureUsable(): void {\n    assert(!this.disposed, 'UI observation has been disposed');\n  }\n\n  private ensureFixedWindowOptions(options?: { domIncluded?: unknown }): void {\n    assert(\n      options?.domIncluded === undefined,\n      'UIObservation does not support domIncluded because it only evaluates recorded screenshots',\n    );\n  }\n\n  async aiQuery<ReturnType = any>(\n    demand: ServiceExtractParam,\n    options?: ObservationQueryOptions,\n  ): Promise<ReturnType> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiQuery<ReturnType>(demand, options);\n  }\n\n  async aiBoolean(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<boolean> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiBoolean(prompt, options);\n  }\n\n  async aiNumber(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<number> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiNumber(prompt, options);\n  }\n\n  async aiString(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<string> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiString(prompt, options);\n  }\n\n  async aiAsk(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<string> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiAsk(prompt, options);\n  }\n\n  async aiAssert(\n    assertion: TUserPrompt,\n    message?: string,\n    options?: ObservationAssertOptions,\n  ): Promise<AgentAssertResult | undefined> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiAssert(assertion, message, options);\n  }\n\n  /** @internal Used only by the CLI observation artifact adapter. */\n  async exportRecord(): Promise<UIObservationRecord> {\n    this.ensureUsable();\n    return cloneUIObservationRecord(this.record);\n  }\n\n  async dispose(): Promise<void> {\n    if (this.disposed) return;\n    this.disposeRecord?.();\n    this.disposed = true;\n    this.onDisposed?.();\n  }\n}\n\ninterface BufferedFrame extends DeviceFrameRef {\n  /** Present once the frame no longer needs to retain an in-memory data URL. */\n  persisted?: UIObservationFrame;\n}\n\nfunction isImageDataUrl(value: unknown): value is string {\n  return (\n    typeof value === 'string' &&\n    /^data:image\\/(?:png|jpe?g);base64,/i.test(value)\n  );\n}\n\n/**\n * Observe an explicit screen window and produce a fixed UIObservation.\n */\nexport class UIObserverImpl implements UIObserver {\n  private frames: BufferedFrame[] = [];\n  private source: DeviceFrameSource | null = null;\n  private usingFallback = false;\n  private stopped = false;\n  private disposed = false;\n  private loopPromise: Promise<void> | null = null;\n  private stopPromise: Promise<UIObservationImpl> | null = null;\n  private representative: UIContext | null = null;\n  private representativeFrame: UIObservationFrame | null = null;\n  private watchdogTimer: ReturnType<typeof setTimeout> | null = null;\n  private persistPromise: Promise<void> | null = null;\n  private persistedByRef = new Map<unknown, UIObservationFrame>();\n  private observation: UIObservationImpl | null = null;\n  private startedAt = 0;\n  private readonly intervalMs: number;\n  private readonly maxFrames: number;\n  private readonly watchdogMs: number;\n  private readonly screenshotShrinkFactor: number;\n  private readonly writer: UIObservationRecordWriter;\n\n  constructor(\n    private readonly deps: UIObserverDeps,\n    opt?: UIObserverOption,\n  ) {\n    this.intervalMs = Math.max(\n      MIN_INTERVAL_MS,\n      opt?.intervalMs ?? DEFAULT_INTERVAL_MS,\n    );\n    this.maxFrames = Math.max(2, opt?.maxFrames ?? DEFAULT_MAX_FRAMES);\n    this.watchdogMs = opt?.watchdogMs ?? DEFAULT_WATCHDOG_MS;\n    this.screenshotShrinkFactor = deps.screenshotShrinkFactor ?? 1;\n    this.writer =\n      deps.observationRecordWriter ?? new UIObservationRecordWriter();\n  }\n\n  get bufferedFrameCount(): number {\n    return this.frames.length;\n  }\n\n  async start(): Promise<void> {\n    assert(!this.loopPromise && !this.stopped, 'observer has already started');\n    this.startedAt = Date.now();\n    try {\n      this.source = (await this.deps.openFrameSource()) ?? null;\n    } catch (error) {\n      debug(`frame source unavailable, using screenshot fallback: ${error}`);\n      this.source = null;\n    }\n    this.usingFallback = !this.source;\n    if (this.usingFallback) {\n      debug('no continuous frame source; sampling via plain screenshots');\n    } else {\n      const waitStart = Date.now();\n      while (\n        !this.source!.latest() &&\n        Date.now() - waitStart < FIRST_FRAME_TIMEOUT_MS\n      ) {\n        await new Promise((resolve) => setTimeout(resolve, 50));\n      }\n      if (!this.source!.latest()) {\n        debug(\n          `no first frame within ${FIRST_FRAME_TIMEOUT_MS}ms; starting anyway`,\n        );\n      }\n    }\n    await this.captureOnce();\n    this.loopPromise = this.runLoop();\n\n    if (this.watchdogMs > 0) {\n      this.watchdogTimer = setTimeout(() => {\n        warnObserver(\n          `UIObserver auto-stopped after ${this.watchdogMs}ms. Call observer.stop() explicitly to avoid this.`,\n        );\n        this.stop().catch(() => {});\n      }, this.watchdogMs);\n      if (\n        typeof (this.watchdogTimer as { unref?: () => void }).unref ===\n        'function'\n      ) {\n        (this.watchdogTimer as { unref: () => void }).unref();\n      }\n    }\n  }\n\n  stop(): Promise<UIObservationImpl> {\n    if (!this.stopPromise) {\n      this.stopPromise = this.finalizeStop();\n    }\n    return this.stopPromise;\n  }\n\n  private async finalizeStop(): Promise<UIObservationImpl> {\n    this.stopped = true;\n    if (this.watchdogTimer) {\n      clearTimeout(this.watchdogTimer);\n      this.watchdogTimer = null;\n    }\n    await this.loopPromise;\n\n    try {\n      if (this.frames.length > 0) {\n        this.persistPromise = this.persistUnstoredFrames().catch((error) => {\n          debug(`frame persistence failed, will retry during export: ${error}`);\n        });\n      }\n      const representativePromise = this.deps.captureRepresentative();\n      const [, representative] = await Promise.all([\n        this.persistPromise,\n        representativePromise,\n      ]);\n\n      const lastFrame = this.frames.at(-1);\n      if (this.source && lastFrame?.persisted) {\n        this.representativeFrame = {\n          ...lastFrame.persisted,\n          capturedAt: lastFrame.capturedAt,\n        };\n        representative.screenshot = ScreenshotItem.fromFile(\n          this.writer.resolveFramePath(lastFrame.persisted),\n          lastFrame.persisted.mimeType,\n          lastFrame.capturedAt,\n        );\n        debug('representative screenshot aligned with last sampled frame');\n      }\n      this.representative = representative;\n      const endedAt = Date.now();\n      const record = await this.finalizeRecord(endedAt);\n      this.observation = new UIObservationImpl(\n        record,\n        this.deps.createInsight(record),\n        () => this.writer.dispose(),\n        this.deps.onDisposed,\n      );\n      return this.observation;\n    } finally {\n      if (this.source) {\n        try {\n          await this.source.stop();\n        } catch (error) {\n          debug(`error stopping frame source: ${error}`);\n        }\n      }\n      debug(\n        `observation stopped with ${this.frames.length} buffered frames (+1 representative)`,\n      );\n      this.deps.onStopped?.();\n    }\n  }\n\n  private async finalizeRecord(endedAt: number): Promise<UIObservationRecord> {\n    assert(\n      this.stopped && this.representative,\n      'observation must be stopped before finalizing the observed window',\n    );\n    if (this.persistPromise) {\n      await this.persistPromise;\n      this.persistPromise = null;\n    }\n    await this.persistUnstoredFrames();\n\n    const sampledFrames = this.frames.map((frame) => {\n      assert(frame.persisted, 'observation frame was not persisted');\n      return { ...frame.persisted, capturedAt: frame.capturedAt };\n    });\n\n    if (!this.representativeFrame) {\n      const representative = this.representative!;\n      this.representativeFrame = this.writer.persistFrame(\n        representative.screenshot.base64,\n        representative.screenshot.capturedAt,\n      );\n      representative.screenshot = ScreenshotItem.fromFile(\n        this.writer.resolveFramePath(this.representativeFrame),\n        this.representativeFrame.mimeType,\n        this.representativeFrame.capturedAt,\n      );\n    }\n\n    const frames = [...sampledFrames, this.representativeFrame];\n    if (frames.length > MAX_FRAMES_PER_RECORD) {\n      warnObserver(\n        `WARNING: exporting ${frames.length} frames (soft limit ${MAX_FRAMES_PER_RECORD}). Running insight against this observation sends every frame to the model; consider increasing intervalMs or decreasing maxFrames to reduce token cost.`,\n      );\n    }\n    debug(\n      `exporting ${frames.length} file-backed observation frames (${this.persistedByRef.size} decoded source refs)`,\n    );\n    const metadata: UIObservationRecordMetadata = {\n      startedAt: this.startedAt,\n      endedAt,\n      shotSize: { ...this.representative!.shotSize },\n      shrunkShotToLogicalRatio: this.representative!.shrunkShotToLogicalRatio,\n    };\n    return this.writer.finalize(frames, metadata);\n  }\n\n  /** Release writer-owned image files after the observation is no longer needed. */\n  async dispose(): Promise<void> {\n    if (this.disposed) return;\n    try {\n      if (this.stopPromise) {\n        await this.stopPromise;\n      } else if (!this.stopped) {\n        await this.stop();\n      }\n    } finally {\n      if (this.observation) {\n        await this.observation.dispose();\n      } else {\n        this.writer.dispose();\n        this.deps.onDisposed?.();\n      }\n      this.frames = [];\n      this.persistedByRef.clear();\n      this.observation = null;\n      this.disposed = true;\n    }\n  }\n\n  private async persistUnstoredFrames(): Promise<void> {\n    const uniqueFrames = this.dedupeRefs(\n      this.frames.filter((frame) => !frame.persisted),\n    );\n    const uncachedFrames = uniqueFrames.filter(\n      (frame) => !this.persistedByRef.has(frame.ref),\n    );\n\n    for (\n      let start = 0;\n      start < uncachedFrames.length;\n      start += DECODE_BATCH_SIZE\n    ) {\n      const batch = uncachedFrames.slice(start, start + DECODE_BATCH_SIZE);\n      const decoded = this.source\n        ? await this.source.decode(batch)\n        : batch.map((frame) => {\n            assert(\n              isImageDataUrl(frame.ref),\n              'fallback observation frame must be an image data URL',\n            );\n            return frame.ref;\n          });\n      assert(\n        decoded.length === batch.length,\n        'frame source decode() must return one image per frame handle',\n      );\n      for (let index = 0; index < batch.length; index++) {\n        const dataUrl = await this.shrinkIfNeeded(decoded[index]);\n        this.persistedByRef.set(\n          batch[index].ref,\n          this.writer.persistFrame(dataUrl, batch[index].capturedAt),\n        );\n      }\n    }\n\n    for (const frame of this.frames) {\n      frame.persisted ??= this.persistedByRef.get(frame.ref);\n    }\n    if (uncachedFrames.length > 0) {\n      debug(`decoded and persisted ${uncachedFrames.length} source frames`);\n    }\n  }\n\n  private async captureOnce(): Promise<void> {\n    try {\n      if (this.source) {\n        const frame = this.source.latest();\n        if (!frame) return;\n        if (isImageDataUrl(frame.ref)) {\n          const dataUrl = await this.shrinkIfNeeded(frame.ref);\n          const persisted = this.writer.persistFrame(dataUrl, frame.capturedAt);\n          this.pushFrame({\n            ref: persisted.path,\n            capturedAt: frame.capturedAt,\n            persisted,\n          });\n        } else {\n          this.pushFrame(frame);\n        }\n        return;\n      }\n      const dataUrl = await this.shrinkIfNeeded(await this.deps.screenshot());\n      const persisted = this.writer.persistFrame(dataUrl, Date.now());\n      this.pushFrame({\n        ref: persisted.path,\n        capturedAt: persisted.capturedAt,\n        persisted,\n      });\n    } catch (error) {\n      debug(`frame capture failed, skipping tick: ${error}`);\n    }\n  }\n\n  private async shrinkIfNeeded(dataUrl: string): Promise<string> {\n    if (this.screenshotShrinkFactor <= 1) return dataUrl;\n    const { width, height } = await imageInfoOfBase64(dataUrl);\n    return resizeImgBase64(dataUrl, {\n      width: Math.round(width / this.screenshotShrinkFactor),\n      height: Math.round(height / this.screenshotShrinkFactor),\n    });\n  }\n\n  private async runLoop(): Promise<void> {\n    while (!this.stopped) {\n      const tickStart = Date.now();\n      await this.captureOnce();\n      while (!this.stopped && Date.now() - tickStart < this.intervalMs) {\n        await new Promise((resolve) => setTimeout(resolve, 50));\n      }\n    }\n  }\n\n  private pushFrame(frame: DeviceFrameRef | BufferedFrame): void {\n    this.frames.push(frame);\n    if (this.frames.length > this.maxFrames) {\n      this.frames = this.thinBuffer(this.frames);\n      this.writer.pruneFrames(\n        this.frames.flatMap((retainedFrame) =>\n          retainedFrame.persisted ? [retainedFrame.persisted] : [],\n        ),\n      );\n      debug(`frame buffer thinned to ${this.frames.length} frames`);\n    }\n  }\n\n  private thinBuffer(frames: BufferedFrame[]): BufferedFrame[] {\n    if (frames.length <= 1) return frames;\n    const isChangePoint = new Array(frames.length).fill(false);\n    isChangePoint[0] = true;\n    for (let index = 1; index < frames.length; index++) {\n      if (frames[index].ref !== frames[index - 1].ref) {\n        isChangePoint[index] = true;\n      }\n    }\n    isChangePoint[frames.length - 1] = true;\n\n    let result: BufferedFrame[] = [];\n    let staticCounter = 0;\n    for (let index = 0; index < frames.length; index++) {\n      if (isChangePoint[index]) {\n        result.push(frames[index]);\n        staticCounter = 0;\n      } else if (staticCounter % 2 === 0) {\n        result.push(frames[index]);\n        staticCounter++;\n      } else {\n        staticCounter++;\n      }\n    }\n\n    if (result.length > this.maxFrames) {\n      const step = result.length / this.maxFrames;\n      const sampled: BufferedFrame[] = [];\n      for (let index = 0; index < this.maxFrames; index++) {\n        sampled.push(result[Math.floor(index * step)]);\n      }\n      sampled[this.maxFrames - 1] = result[result.length - 1];\n      result = sampled;\n    }\n    return result;\n  }\n\n  private dedupeRefs(frames: BufferedFrame[]): BufferedFrame[] {\n    const seen = new Set<unknown>();\n    const result: BufferedFrame[] = [];\n    for (const frame of frames) {\n      if (!seen.has(frame.ref)) {\n        seen.add(frame.ref);\n        result.push(frame);\n      }\n    }\n    return result;\n  }\n}\n\n/** Rebuild model-facing temporal context from resolved image file paths. */\nexport function uiContextFromObservationRecord(\n  record: UIObservationRecord,\n): UIContext {\n  assert(\n    record.type === 'midscene_ui_observation' && record.version === 1,\n    'invalid UI observation record type or version',\n  );\n  assert(record.frames.length > 0, 'UI observation record contains no frames');\n  assert(\n    Number.isFinite(record.shotSize.width) && record.shotSize.width > 0,\n    'UI observation record shot width must be positive',\n  );\n  assert(\n    Number.isFinite(record.shotSize.height) && record.shotSize.height > 0,\n    'UI observation record shot height must be positive',\n  );\n  assert(\n    Number.isFinite(record.shrunkShotToLogicalRatio) &&\n      record.shrunkShotToLogicalRatio > 0,\n    'UI observation record screenshot ratio must be positive',\n  );\n  const screenshotSequence = record.frames.map((frame) =>\n    ScreenshotItem.fromFile(frame.path, frame.mimeType, frame.capturedAt),\n  );\n  return {\n    screenshot: screenshotSequence[screenshotSequence.length - 1],\n    screenshotSequence,\n    shotSize: { ...record.shotSize },\n    shrunkShotToLogicalRatio: record.shrunkShotToLogicalRatio,\n  };\n}\n"],"names":["debug","getDebug","warnObserver","DEFAULT_INTERVAL_MS","MIN_INTERVAL_MS","DEFAULT_MAX_FRAMES","FIRST_FRAME_TIMEOUT_MS","DEFAULT_WATCHDOG_MS","MAX_FRAMES_PER_RECORD","DECODE_BATCH_SIZE","UIObservationImpl","assert","options","undefined","demand","prompt","assertion","message","cloneUIObservationRecord","record","insight","disposeRecord","onDisposed","isImageDataUrl","value","UIObserverImpl","Date","error","waitStart","Promise","resolve","setTimeout","clearTimeout","representativePromise","representative","lastFrame","ScreenshotItem","endedAt","sampledFrames","frame","frames","metadata","uniqueFrames","uncachedFrames","start","batch","decoded","index","dataUrl","persisted","width","height","imageInfoOfBase64","resizeImgBase64","Math","tickStart","retainedFrame","isChangePoint","Array","result","staticCounter","step","sampled","seen","Set","deps","opt","Map","UIObservationRecordWriter","uiContextFromObservationRecord","Number","screenshotSequence"],"mappings":";;;;;;;;;;;;;;;AAyBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,eAAeD,SAAS,eAAe;IAAE,SAAS;AAAK;AAE7D,MAAME,sBAAsB;AAC5B,MAAMC,kBAAkB;AACxB,MAAMC,qBAAqB;AAC3B,MAAMC,yBAAyB;AAC/B,MAAMC,sBAAsB;AAC5B,MAAMC,wBAAwB;AAC9B,MAAMC,oBAAoB;AAyCnB,MAAMC;IAaX,IAAI,aAAqB;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM;IAClC;IAEA,IAAI,YAAoB;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS;IAC9B;IAEA,IAAI,UAAkB;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO;IAC5B;IAEQ,eAAqB;QAC3BC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE;IACzB;IAEQ,yBAAyBC,OAAmC,EAAQ;QAC1ED,OACEC,SAAS,gBAAgBC,QACzB;IAEJ;IAEA,MAAM,QACJC,MAA2B,EAC3BF,OAAiC,EACZ;QACrB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAaE,QAAQF;IAClD;IAEA,MAAM,UACJG,MAAmB,EACnBH,OAAiC,EACf;QAClB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAACG,QAAQH;IACxC;IAEA,MAAM,SACJG,MAAmB,EACnBH,OAAiC,EAChB;QACjB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACG,QAAQH;IACvC;IAEA,MAAM,SACJG,MAAmB,EACnBH,OAAiC,EAChB;QACjB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACG,QAAQH;IACvC;IAEA,MAAM,MACJG,MAAmB,EACnBH,OAAiC,EAChB;QACjB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAACG,QAAQH;IACpC;IAEA,MAAM,SACJI,SAAsB,EACtBC,OAAgB,EAChBL,OAAkC,EACM;QACxC,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACI,WAAWC,SAASL;IACnD;IAGA,MAAM,eAA6C;QACjD,IAAI,CAAC,YAAY;QACjB,OAAOM,yBAAyB,IAAI,CAAC,MAAM;IAC7C;IAEA,MAAM,UAAyB;QAC7B,IAAI,IAAI,CAAC,QAAQ,EAAE;QACnB,IAAI,CAAC,aAAa;QAClB,IAAI,CAAC,QAAQ,GAAG;QAChB,IAAI,CAAC,UAAU;IACjB;IAlGA,YACEC,MAA2B,EACVC,OAAmB,EACnBC,aAA0B,EAC1BC,UAAuB,CACxC;;;;QARF,uBAAQ,YAAR;QACA,uBAAiB,UAAjB;aAImBF,OAAO,GAAPA;aACAC,aAAa,GAAbA;aACAC,UAAU,GAAVA;aAPX,QAAQ,GAAG;QASjB,IAAI,CAAC,MAAM,GAAGJ,yBAAyBC;IACzC;AA4FF;AAOA,SAASI,eAAeC,KAAc;IACpC,OACE,AAAiB,YAAjB,OAAOA,SACP,sCAAsC,IAAI,CAACA;AAE/C;AAKO,MAAMC;IAoCX,IAAI,qBAA6B;QAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM;IAC3B;IAEA,MAAM,QAAuB;QAC3Bd,OAAO,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QAC3C,IAAI,CAAC,SAAS,GAAGe,KAAK,GAAG;QACzB,IAAI;YACF,IAAI,CAAC,MAAM,GAAI,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,MAAO;QACvD,EAAE,OAAOC,OAAO;YACd3B,MAAM,CAAC,qDAAqD,EAAE2B,OAAO;YACrE,IAAI,CAAC,MAAM,GAAG;QAChB;QACA,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,MAAM;QACjC,IAAI,IAAI,CAAC,aAAa,EACpB3B,MAAM;aACD;YACL,MAAM4B,YAAYF,KAAK,GAAG;YAC1B,MACE,CAAC,IAAI,CAAC,MAAM,CAAE,MAAM,MACpBA,KAAK,GAAG,KAAKE,YAAYtB,uBAEzB,MAAM,IAAIuB,QAAQ,CAACC,UAAYC,WAAWD,SAAS;YAErD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAE,MAAM,IACtB9B,MACE,CAAC,sBAAsB,EAAEM,uBAAuB,mBAAmB,CAAC;QAG1E;QACA,MAAM,IAAI,CAAC,WAAW;QACtB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO;QAE/B,IAAI,IAAI,CAAC,UAAU,GAAG,GAAG;YACvB,IAAI,CAAC,aAAa,GAAGyB,WAAW;gBAC9B7B,aACE,CAAC,8BAA8B,EAAE,IAAI,CAAC,UAAU,CAAC,kDAAkD,CAAC;gBAEtG,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,KAAO;YAC3B,GAAG,IAAI,CAAC,UAAU;YAClB,IACE,AACA,cADA,OAAQ,IAAI,CAAC,aAAa,CAA4B,KAAK,EAG1D,IAAI,CAAC,aAAa,CAA2B,KAAK;QAEvD;IACF;IAEA,OAAmC;QACjC,IAAI,CAAC,IAAI,CAAC,WAAW,EACnB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY;QAEtC,OAAO,IAAI,CAAC,WAAW;IACzB;IAEA,MAAc,eAA2C;QACvD,IAAI,CAAC,OAAO,GAAG;QACf,IAAI,IAAI,CAAC,aAAa,EAAE;YACtB8B,aAAa,IAAI,CAAC,aAAa;YAC/B,IAAI,CAAC,aAAa,GAAG;QACvB;QACA,MAAM,IAAI,CAAC,WAAW;QAEtB,IAAI;YACF,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,GACvB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC,CAACL;gBACxD3B,MAAM,CAAC,oDAAoD,EAAE2B,OAAO;YACtE;YAEF,MAAMM,wBAAwB,IAAI,CAAC,IAAI,CAAC,qBAAqB;YAC7D,MAAM,GAAGC,eAAe,GAAG,MAAML,QAAQ,GAAG,CAAC;gBAC3C,IAAI,CAAC,cAAc;gBACnBI;aACD;YAED,MAAME,YAAY,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,MAAM,IAAIA,WAAW,WAAW;gBACvC,IAAI,CAAC,mBAAmB,GAAG;oBACzB,GAAGA,UAAU,SAAS;oBACtB,YAAYA,UAAU,UAAU;gBAClC;gBACAD,eAAe,UAAU,GAAGE,eAAe,QAAQ,CACjD,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAACD,UAAU,SAAS,GAChDA,UAAU,SAAS,CAAC,QAAQ,EAC5BA,UAAU,UAAU;gBAEtBnC,MAAM;YACR;YACA,IAAI,CAAC,cAAc,GAAGkC;YACtB,MAAMG,UAAUX,KAAK,GAAG;YACxB,MAAMP,SAAS,MAAM,IAAI,CAAC,cAAc,CAACkB;YACzC,IAAI,CAAC,WAAW,GAAG,IAAI3B,kBACrBS,QACA,IAAI,CAAC,IAAI,CAAC,aAAa,CAACA,SACxB,IAAM,IAAI,CAAC,MAAM,CAAC,OAAO,IACzB,IAAI,CAAC,IAAI,CAAC,UAAU;YAEtB,OAAO,IAAI,CAAC,WAAW;QACzB,SAAU;YACR,IAAI,IAAI,CAAC,MAAM,EACb,IAAI;gBACF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI;YACxB,EAAE,OAAOQ,OAAO;gBACd3B,MAAM,CAAC,6BAA6B,EAAE2B,OAAO;YAC/C;YAEF3B,MACE,CAAC,yBAAyB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,oCAAoC,CAAC;YAEtF,IAAI,CAAC,IAAI,CAAC,SAAS;QACrB;IACF;IAEA,MAAc,eAAeqC,OAAe,EAAgC;QAC1E1B,OACE,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,EACnC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,MAAM,IAAI,CAAC,cAAc;YACzB,IAAI,CAAC,cAAc,GAAG;QACxB;QACA,MAAM,IAAI,CAAC,qBAAqB;QAEhC,MAAM2B,gBAAgB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAACC;YACrC5B,OAAO4B,MAAM,SAAS,EAAE;YACxB,OAAO;gBAAE,GAAGA,MAAM,SAAS;gBAAE,YAAYA,MAAM,UAAU;YAAC;QAC5D;QAEA,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,MAAML,iBAAiB,IAAI,CAAC,cAAc;YAC1C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CACjDA,eAAe,UAAU,CAAC,MAAM,EAChCA,eAAe,UAAU,CAAC,UAAU;YAEtCA,eAAe,UAAU,GAAGE,eAAe,QAAQ,CACjD,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,mBAAmB,GACrD,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EACjC,IAAI,CAAC,mBAAmB,CAAC,UAAU;QAEvC;QAEA,MAAMI,SAAS;eAAIF;YAAe,IAAI,CAAC,mBAAmB;SAAC;QAC3D,IAAIE,OAAO,MAAM,GAAGhC,uBAClBN,aACE,CAAC,mBAAmB,EAAEsC,OAAO,MAAM,CAAC,oBAAoB,EAAEhC,sBAAsB,wJAAwJ,CAAC;QAG7OR,MACE,CAAC,UAAU,EAAEwC,OAAO,MAAM,CAAC,iCAAiC,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAE/G,MAAMC,WAAwC;YAC5C,WAAW,IAAI,CAAC,SAAS;YACzBJ;YACA,UAAU;gBAAE,GAAG,IAAI,CAAC,cAAc,CAAE,QAAQ;YAAC;YAC7C,0BAA0B,IAAI,CAAC,cAAc,CAAE,wBAAwB;QACzE;QACA,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAACG,QAAQC;IACtC;IAGA,MAAM,UAAyB;QAC7B,IAAI,IAAI,CAAC,QAAQ,EAAE;QACnB,IAAI;YACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW;iBACjB,IAAI,CAAC,IAAI,CAAC,OAAO,EACtB,MAAM,IAAI,CAAC,IAAI;QAEnB,SAAU;YACR,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO;iBACzB;gBACL,IAAI,CAAC,MAAM,CAAC,OAAO;gBACnB,IAAI,CAAC,IAAI,CAAC,UAAU;YACtB;YACA,IAAI,CAAC,MAAM,GAAG,EAAE;YAChB,IAAI,CAAC,cAAc,CAAC,KAAK;YACzB,IAAI,CAAC,WAAW,GAAG;YACnB,IAAI,CAAC,QAAQ,GAAG;QAClB;IACF;IAEA,MAAc,wBAAuC;QACnD,MAAMC,eAAe,IAAI,CAAC,UAAU,CAClC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAACH,QAAU,CAACA,MAAM,SAAS;QAEhD,MAAMI,iBAAiBD,aAAa,MAAM,CACxC,CAACH,QAAU,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAACA,MAAM,GAAG;QAG/C,IACE,IAAIK,QAAQ,GACZA,QAAQD,eAAe,MAAM,EAC7BC,SAASnC,kBACT;YACA,MAAMoC,QAAQF,eAAe,KAAK,CAACC,OAAOA,QAAQnC;YAClD,MAAMqC,UAAU,IAAI,CAAC,MAAM,GACvB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAACD,SACzBA,MAAM,GAAG,CAAC,CAACN;gBACT5B,OACEY,eAAegB,MAAM,GAAG,GACxB;gBAEF,OAAOA,MAAM,GAAG;YAClB;YACJ5B,OACEmC,QAAQ,MAAM,KAAKD,MAAM,MAAM,EAC/B;YAEF,IAAK,IAAIE,QAAQ,GAAGA,QAAQF,MAAM,MAAM,EAAEE,QAAS;gBACjD,MAAMC,UAAU,MAAM,IAAI,CAAC,cAAc,CAACF,OAAO,CAACC,MAAM;gBACxD,IAAI,CAAC,cAAc,CAAC,GAAG,CACrBF,KAAK,CAACE,MAAM,CAAC,GAAG,EAChB,IAAI,CAAC,MAAM,CAAC,YAAY,CAACC,SAASH,KAAK,CAACE,MAAM,CAAC,UAAU;YAE7D;QACF;QAEA,KAAK,MAAMR,SAAS,IAAI,CAAC,MAAM,CAC7BA,MAAM,SAAS,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,CAACA,MAAM,GAAG;QAEvD,IAAII,eAAe,MAAM,GAAG,GAC1B3C,MAAM,CAAC,sBAAsB,EAAE2C,eAAe,MAAM,CAAC,cAAc,CAAC;IAExE;IAEA,MAAc,cAA6B;QACzC,IAAI;YACF,IAAI,IAAI,CAAC,MAAM,EAAE;gBACf,MAAMJ,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM;gBAChC,IAAI,CAACA,OAAO;gBACZ,IAAIhB,eAAegB,MAAM,GAAG,GAAG;oBAC7B,MAAMS,UAAU,MAAM,IAAI,CAAC,cAAc,CAACT,MAAM,GAAG;oBACnD,MAAMU,YAAY,IAAI,CAAC,MAAM,CAAC,YAAY,CAACD,SAAST,MAAM,UAAU;oBACpE,IAAI,CAAC,SAAS,CAAC;wBACb,KAAKU,UAAU,IAAI;wBACnB,YAAYV,MAAM,UAAU;wBAC5BU;oBACF;gBACF,OACE,IAAI,CAAC,SAAS,CAACV;gBAEjB;YACF;YACA,MAAMS,UAAU,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU;YACpE,MAAMC,YAAY,IAAI,CAAC,MAAM,CAAC,YAAY,CAACD,SAAStB,KAAK,GAAG;YAC5D,IAAI,CAAC,SAAS,CAAC;gBACb,KAAKuB,UAAU,IAAI;gBACnB,YAAYA,UAAU,UAAU;gBAChCA;YACF;QACF,EAAE,OAAOtB,OAAO;YACd3B,MAAM,CAAC,qCAAqC,EAAE2B,OAAO;QACvD;IACF;IAEA,MAAc,eAAeqB,OAAe,EAAmB;QAC7D,IAAI,IAAI,CAAC,sBAAsB,IAAI,GAAG,OAAOA;QAC7C,MAAM,EAAEE,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMC,kBAAkBJ;QAClD,OAAOK,gBAAgBL,SAAS;YAC9B,OAAOM,KAAK,KAAK,CAACJ,QAAQ,IAAI,CAAC,sBAAsB;YACrD,QAAQI,KAAK,KAAK,CAACH,SAAS,IAAI,CAAC,sBAAsB;QACzD;IACF;IAEA,MAAc,UAAyB;QACrC,MAAO,CAAC,IAAI,CAAC,OAAO,CAAE;YACpB,MAAMI,YAAY7B,KAAK,GAAG;YAC1B,MAAM,IAAI,CAAC,WAAW;YACtB,MAAO,CAAC,IAAI,CAAC,OAAO,IAAIA,KAAK,GAAG,KAAK6B,YAAY,IAAI,CAAC,UAAU,CAC9D,MAAM,IAAI1B,QAAQ,CAACC,UAAYC,WAAWD,SAAS;QAEvD;IACF;IAEQ,UAAUS,KAAqC,EAAQ;QAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CAACA;QACjB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;YACvC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM;YACzC,IAAI,CAAC,MAAM,CAAC,WAAW,CACrB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAACiB,gBACnBA,cAAc,SAAS,GAAG;oBAACA,cAAc,SAAS;iBAAC,GAAG,EAAE;YAG5DxD,MAAM,CAAC,wBAAwB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;QAC9D;IACF;IAEQ,WAAWwC,MAAuB,EAAmB;QAC3D,IAAIA,OAAO,MAAM,IAAI,GAAG,OAAOA;QAC/B,MAAMiB,gBAAgB,IAAIC,MAAMlB,OAAO,MAAM,EAAE,IAAI,CAAC;QACpDiB,aAAa,CAAC,EAAE,GAAG;QACnB,IAAK,IAAIV,QAAQ,GAAGA,QAAQP,OAAO,MAAM,EAAEO,QACzC,IAAIP,MAAM,CAACO,MAAM,CAAC,GAAG,KAAKP,MAAM,CAACO,QAAQ,EAAE,CAAC,GAAG,EAC7CU,aAAa,CAACV,MAAM,GAAG;QAG3BU,aAAa,CAACjB,OAAO,MAAM,GAAG,EAAE,GAAG;QAEnC,IAAImB,SAA0B,EAAE;QAChC,IAAIC,gBAAgB;QACpB,IAAK,IAAIb,QAAQ,GAAGA,QAAQP,OAAO,MAAM,EAAEO,QACzC,IAAIU,aAAa,CAACV,MAAM,EAAE;YACxBY,OAAO,IAAI,CAACnB,MAAM,CAACO,MAAM;YACzBa,gBAAgB;QAClB,OAAO,IAAIA,gBAAgB,MAAM,GAAG;YAClCD,OAAO,IAAI,CAACnB,MAAM,CAACO,MAAM;YACzBa;QACF,OACEA;QAIJ,IAAID,OAAO,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;YAClC,MAAME,OAAOF,OAAO,MAAM,GAAG,IAAI,CAAC,SAAS;YAC3C,MAAMG,UAA2B,EAAE;YACnC,IAAK,IAAIf,QAAQ,GAAGA,QAAQ,IAAI,CAAC,SAAS,EAAEA,QAC1Ce,QAAQ,IAAI,CAACH,MAAM,CAACL,KAAK,KAAK,CAACP,QAAQc,MAAM;YAE/CC,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,EAAE,GAAGH,MAAM,CAACA,OAAO,MAAM,GAAG,EAAE;YACvDA,SAASG;QACX;QACA,OAAOH;IACT;IAEQ,WAAWnB,MAAuB,EAAmB;QAC3D,MAAMuB,OAAO,IAAIC;QACjB,MAAML,SAA0B,EAAE;QAClC,KAAK,MAAMpB,SAASC,OAClB,IAAI,CAACuB,KAAK,GAAG,CAACxB,MAAM,GAAG,GAAG;YACxBwB,KAAK,GAAG,CAACxB,MAAM,GAAG;YAClBoB,OAAO,IAAI,CAACpB;QACd;QAEF,OAAOoB;IACT;IAhWA,YACmBM,IAAoB,EACrCC,GAAsB,CACtB;;QAvBF,uBAAQ,UAAR;QACA,uBAAQ,UAAR;QACA,uBAAQ,iBAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,YAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,kBAAR;QACA,uBAAQ,uBAAR;QACA,uBAAQ,iBAAR;QACA,uBAAQ,kBAAR;QACA,uBAAQ,kBAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,aAAR;QACA,uBAAiB,cAAjB;QACA,uBAAiB,aAAjB;QACA,uBAAiB,cAAjB;QACA,uBAAiB,0BAAjB;QACA,uBAAiB,UAAjB;aAGmBD,IAAI,GAAJA;aArBX,MAAM,GAAoB,EAAE;aAC5B,MAAM,GAA6B;aACnC,aAAa,GAAG;aAChB,OAAO,GAAG;aACV,QAAQ,GAAG;aACX,WAAW,GAAyB;aACpC,WAAW,GAAsC;aACjD,cAAc,GAAqB;aACnC,mBAAmB,GAA8B;aACjD,aAAa,GAAyC;aACtD,cAAc,GAAyB;aACvC,cAAc,GAAG,IAAIE;aACrB,WAAW,GAA6B;aACxC,SAAS,GAAG;QAWlB,IAAI,CAAC,UAAU,GAAGb,KAAK,GAAG,CACxBlD,iBACA8D,KAAK,cAAc/D;QAErB,IAAI,CAAC,SAAS,GAAGmD,KAAK,GAAG,CAAC,GAAGY,KAAK,aAAa7D;QAC/C,IAAI,CAAC,UAAU,GAAG6D,KAAK,cAAc3D;QACrC,IAAI,CAAC,sBAAsB,GAAG0D,KAAK,sBAAsB,IAAI;QAC7D,IAAI,CAAC,MAAM,GACTA,KAAK,uBAAuB,IAAI,IAAIG;IACxC;AAoVF;AAGO,SAASC,+BACdlD,MAA2B;IAE3BR,OACEQ,AAAgB,8BAAhBA,OAAO,IAAI,IAAkCA,AAAmB,MAAnBA,OAAO,OAAO,EAC3D;IAEFR,OAAOQ,OAAO,MAAM,CAAC,MAAM,GAAG,GAAG;IACjCR,OACE2D,OAAO,QAAQ,CAACnD,OAAO,QAAQ,CAAC,KAAK,KAAKA,OAAO,QAAQ,CAAC,KAAK,GAAG,GAClE;IAEFR,OACE2D,OAAO,QAAQ,CAACnD,OAAO,QAAQ,CAAC,MAAM,KAAKA,OAAO,QAAQ,CAAC,MAAM,GAAG,GACpE;IAEFR,OACE2D,OAAO,QAAQ,CAACnD,OAAO,wBAAwB,KAC7CA,OAAO,wBAAwB,GAAG,GACpC;IAEF,MAAMoD,qBAAqBpD,OAAO,MAAM,CAAC,GAAG,CAAC,CAACoB,QAC5CH,eAAe,QAAQ,CAACG,MAAM,IAAI,EAAEA,MAAM,QAAQ,EAAEA,MAAM,UAAU;IAEtE,OAAO;QACL,YAAYgC,kBAAkB,CAACA,mBAAmB,MAAM,GAAG,EAAE;QAC7DA;QACA,UAAU;YAAE,GAAGpD,OAAO,QAAQ;QAAC;QAC/B,0BAA0BA,OAAO,wBAAwB;IAC3D;AACF"}
+{"version":3,"file":"agent/ui-observer.mjs","sources":["../../../src/agent/ui-observer.ts"],"sourcesContent":["import {\n  type UIObservationRecordMetadata,\n  UIObservationRecordWriter,\n  cloneUIObservationRecord,\n} from '@midscene/shared/agent-tools/observation-record';\nimport type {\n  BaseUIObserverOptions,\n  UIObservationFrame,\n  UIObservationRecord,\n} from '@midscene/shared/agent-tools/types';\nimport {\n  convertPngBase64ToJpeg,\n  imageInfoOfBase64,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../common';\nimport type { DeviceFrameRef, DeviceFrameSource } from '../device';\nimport { ScreenshotItem } from '../screenshot-item';\nimport type {\n  AgentAssertResult,\n  InsightAPI,\n  ObservationAssertOptions,\n  ObservationQueryOptions,\n  ServiceExtractParam,\n  UIContext,\n} from '../types';\n\nconst debug = getDebug('ui-observer');\nconst warnObserver = getDebug('ui-observer', { console: true });\n\nconst DEFAULT_INTERVAL_MS = 1000;\nconst MIN_INTERVAL_MS = 200;\nconst DEFAULT_MAX_FRAMES = 30;\nconst FIRST_FRAME_TIMEOUT_MS = 3000;\nconst DEFAULT_WATCHDOG_MS = 5 * 60 * 1000;\nconst MAX_FRAMES_PER_RECORD = 50;\nconst DECODE_BATCH_SIZE = 4;\nconst OBSERVATION_JPEG_QUALITY = 90;\n\n/** Options for a UI observation window. */\nexport type UIObserverOption = BaseUIObserverOptions;\n\ninterface UIObserverDeps {\n  openFrameSource: () => Promise<DeviceFrameSource | undefined>;\n  screenshot: () => Promise<string>;\n  captureRepresentative: () => Promise<UIContext>;\n  createInsight: (record: UIObservationRecord) => InsightAPI;\n  onStopped?: () => void;\n  onDisposed?: () => void;\n  screenshotShrinkFactor?: number;\n  /** Test/internal persistence override; not part of the Agent SDK options. */\n  observationRecordWriter?: UIObservationRecordWriter;\n}\n\n/** A fixed screen-recording window that supports read-only AI insights. */\nexport interface UIObservation\n  extends InsightAPI<ObservationQueryOptions, ObservationAssertOptions> {\n  /** Number of captured frames in the fixed observation window. */\n  readonly frameCount: number;\n  /** Timestamp when screen sampling started. */\n  readonly startedAt: number;\n  /** Timestamp when screen sampling ended. */\n  readonly endedAt: number;\n  /** Release image files owned by this observation. Failed cleanup is retryable. */\n  dispose(): Promise<void>;\n}\n\n/** Recording lifecycle returned by {@link Agent.startObserving}. */\nexport interface UIObserver {\n  /** Number of frames currently buffered while recording. */\n  readonly bufferedFrameCount: number;\n  /** Stop recording and return its fixed observation window. */\n  stop(): Promise<UIObservation>;\n  /** Stop recording if needed and release its image files. */\n  dispose(): Promise<void>;\n}\n\n/** @internal Concrete fixed-window implementation. */\nexport class UIObservationImpl implements UIObservation {\n  private disposed = false;\n  private readonly record: UIObservationRecord;\n\n  constructor(\n    record: UIObservationRecord,\n    private readonly insight: InsightAPI,\n    private readonly disposeRecord?: () => void,\n    private readonly onDisposed?: () => void,\n  ) {\n    this.record = cloneUIObservationRecord(record);\n  }\n\n  get frameCount(): number {\n    return this.record.frames.length;\n  }\n\n  get startedAt(): number {\n    return this.record.startedAt;\n  }\n\n  get endedAt(): number {\n    return this.record.endedAt;\n  }\n\n  private ensureUsable(): void {\n    assert(!this.disposed, 'UI observation has been disposed');\n  }\n\n  private ensureFixedWindowOptions(options?: { domIncluded?: unknown }): void {\n    assert(\n      options?.domIncluded === undefined,\n      'UIObservation does not support domIncluded because it only evaluates recorded screenshots',\n    );\n  }\n\n  async aiQuery<ReturnType = any>(\n    demand: ServiceExtractParam,\n    options?: ObservationQueryOptions,\n  ): Promise<ReturnType> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiQuery<ReturnType>(demand, options);\n  }\n\n  async aiBoolean(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<boolean> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiBoolean(prompt, options);\n  }\n\n  async aiNumber(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<number> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiNumber(prompt, options);\n  }\n\n  async aiString(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<string> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiString(prompt, options);\n  }\n\n  async aiAsk(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<string> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiAsk(prompt, options);\n  }\n\n  async aiAssert(\n    assertion: TUserPrompt,\n    message?: string,\n    options?: ObservationAssertOptions,\n  ): Promise<AgentAssertResult | undefined> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiAssert(assertion, message, options);\n  }\n\n  /** @internal Used only by the CLI observation artifact adapter. */\n  async exportRecord(): Promise<UIObservationRecord> {\n    this.ensureUsable();\n    return cloneUIObservationRecord(this.record);\n  }\n\n  async dispose(): Promise<void> {\n    if (this.disposed) return;\n    this.disposeRecord?.();\n    this.disposed = true;\n    this.onDisposed?.();\n  }\n}\n\ninterface BufferedFrame extends DeviceFrameRef {\n  /** Present once the frame no longer needs to retain an in-memory data URL. */\n  persisted?: UIObservationFrame;\n}\n\nfunction isImageDataUrl(value: unknown): value is string {\n  return (\n    typeof value === 'string' &&\n    /^data:image\\/(?:png|jpe?g);base64,/i.test(value)\n  );\n}\n\n/**\n * Observe an explicit screen window and produce a fixed UIObservation.\n */\nexport class UIObserverImpl implements UIObserver {\n  private frames: BufferedFrame[] = [];\n  private source: DeviceFrameSource | null = null;\n  private usingFallback = false;\n  private stopped = false;\n  private disposed = false;\n  private loopPromise: Promise<void> | null = null;\n  private stopPromise: Promise<UIObservationImpl> | null = null;\n  private representative: UIContext | null = null;\n  private representativeFrame: UIObservationFrame | null = null;\n  private watchdogTimer: ReturnType<typeof setTimeout> | null = null;\n  private persistPromise: Promise<void> | null = null;\n  private persistedByRef = new Map<unknown, UIObservationFrame>();\n  private observation: UIObservationImpl | null = null;\n  private startedAt = 0;\n  private readonly intervalMs: number;\n  private readonly maxFrames: number;\n  private readonly watchdogMs: number;\n  private readonly screenshotShrinkFactor: number;\n  private readonly writer: UIObservationRecordWriter;\n\n  constructor(\n    private readonly deps: UIObserverDeps,\n    opt?: UIObserverOption,\n  ) {\n    this.intervalMs = Math.max(\n      MIN_INTERVAL_MS,\n      opt?.intervalMs ?? DEFAULT_INTERVAL_MS,\n    );\n    this.maxFrames = Math.max(2, opt?.maxFrames ?? DEFAULT_MAX_FRAMES);\n    this.watchdogMs = opt?.watchdogMs ?? DEFAULT_WATCHDOG_MS;\n    this.screenshotShrinkFactor = deps.screenshotShrinkFactor ?? 1;\n    this.writer =\n      deps.observationRecordWriter ?? new UIObservationRecordWriter();\n  }\n\n  get bufferedFrameCount(): number {\n    return this.frames.length;\n  }\n\n  async start(): Promise<void> {\n    assert(!this.loopPromise && !this.stopped, 'observer has already started');\n    this.startedAt = Date.now();\n    try {\n      this.source = (await this.deps.openFrameSource()) ?? null;\n    } catch (error) {\n      debug(`frame source unavailable, using screenshot fallback: ${error}`);\n      this.source = null;\n    }\n    this.usingFallback = !this.source;\n    if (this.usingFallback) {\n      debug('no continuous frame source; sampling via plain screenshots');\n    } else {\n      const waitStart = Date.now();\n      while (\n        !this.source!.latest() &&\n        Date.now() - waitStart < FIRST_FRAME_TIMEOUT_MS\n      ) {\n        await new Promise((resolve) => setTimeout(resolve, 50));\n      }\n      if (!this.source!.latest()) {\n        debug(\n          `no first frame within ${FIRST_FRAME_TIMEOUT_MS}ms; starting anyway`,\n        );\n      }\n    }\n    await this.captureOnce();\n    this.loopPromise = this.runLoop();\n\n    if (this.watchdogMs > 0) {\n      this.watchdogTimer = setTimeout(() => {\n        warnObserver(\n          `UIObserver auto-stopped after ${this.watchdogMs}ms. Call observer.stop() explicitly to avoid this.`,\n        );\n        this.stop().catch(() => {});\n      }, this.watchdogMs);\n      if (\n        typeof (this.watchdogTimer as { unref?: () => void }).unref ===\n        'function'\n      ) {\n        (this.watchdogTimer as { unref: () => void }).unref();\n      }\n    }\n  }\n\n  stop(): Promise<UIObservationImpl> {\n    if (!this.stopPromise) {\n      this.stopPromise = this.finalizeStop();\n    }\n    return this.stopPromise;\n  }\n\n  private async finalizeStop(): Promise<UIObservationImpl> {\n    this.stopped = true;\n    if (this.watchdogTimer) {\n      clearTimeout(this.watchdogTimer);\n      this.watchdogTimer = null;\n    }\n    await this.loopPromise;\n\n    try {\n      if (this.frames.length > 0) {\n        this.persistPromise = this.persistUnstoredFrames().catch((error) => {\n          debug(`frame persistence failed, will retry during export: ${error}`);\n        });\n      }\n      const representativePromise = this.deps.captureRepresentative();\n      const [, representative] = await Promise.all([\n        this.persistPromise,\n        representativePromise,\n      ]);\n\n      const lastFrame = this.frames.at(-1);\n      if (this.source && lastFrame?.persisted) {\n        this.representativeFrame = {\n          ...lastFrame.persisted,\n          capturedAt: lastFrame.capturedAt,\n        };\n        representative.screenshot = ScreenshotItem.fromFile(\n          this.writer.resolveFramePath(lastFrame.persisted),\n          lastFrame.persisted.mimeType,\n          lastFrame.capturedAt,\n        );\n        debug('representative screenshot aligned with last sampled frame');\n      }\n      this.representative = representative;\n      const endedAt = Date.now();\n      const record = await this.finalizeRecord(endedAt);\n      this.observation = new UIObservationImpl(\n        record,\n        this.deps.createInsight(record),\n        () => this.writer.dispose(),\n        this.deps.onDisposed,\n      );\n      return this.observation;\n    } finally {\n      if (this.source) {\n        try {\n          await this.source.stop();\n        } catch (error) {\n          debug(`error stopping frame source: ${error}`);\n        }\n      }\n      debug(\n        `observation stopped with ${this.frames.length} buffered frames (+1 representative)`,\n      );\n      this.deps.onStopped?.();\n    }\n  }\n\n  private async finalizeRecord(endedAt: number): Promise<UIObservationRecord> {\n    assert(\n      this.stopped && this.representative,\n      'observation must be stopped before finalizing the observed window',\n    );\n    if (this.persistPromise) {\n      await this.persistPromise;\n      this.persistPromise = null;\n    }\n    await this.persistUnstoredFrames();\n\n    const sampledFrames = this.frames.map((frame) => {\n      assert(frame.persisted, 'observation frame was not persisted');\n      return { ...frame.persisted, capturedAt: frame.capturedAt };\n    });\n\n    if (!this.representativeFrame) {\n      const representative = this.representative!;\n      this.representativeFrame = this.writer.persistFrame(\n        await this.prepareFrameForPersistence(representative.screenshot.base64),\n        representative.screenshot.capturedAt,\n      );\n      representative.screenshot = ScreenshotItem.fromFile(\n        this.writer.resolveFramePath(this.representativeFrame),\n        this.representativeFrame.mimeType,\n        this.representativeFrame.capturedAt,\n      );\n    }\n\n    const frames = [...sampledFrames, this.representativeFrame];\n    if (frames.length > MAX_FRAMES_PER_RECORD) {\n      warnObserver(\n        `WARNING: exporting ${frames.length} frames (soft limit ${MAX_FRAMES_PER_RECORD}). Running insight against this observation sends every frame to the model; consider increasing intervalMs or decreasing maxFrames to reduce token cost.`,\n      );\n    }\n    debug(\n      `exporting ${frames.length} file-backed observation frames (${this.persistedByRef.size} decoded source refs)`,\n    );\n    const metadata: UIObservationRecordMetadata = {\n      startedAt: this.startedAt,\n      endedAt,\n      shotSize: { ...this.representative!.shotSize },\n      shrunkShotToLogicalRatio: this.representative!.shrunkShotToLogicalRatio,\n    };\n    return this.writer.finalize(frames, metadata);\n  }\n\n  /** Release writer-owned image files after the observation is no longer needed. */\n  async dispose(): Promise<void> {\n    if (this.disposed) return;\n    try {\n      if (this.stopPromise) {\n        await this.stopPromise;\n      } else if (!this.stopped) {\n        await this.stop();\n      }\n    } finally {\n      if (this.observation) {\n        await this.observation.dispose();\n      } else {\n        this.writer.dispose();\n        this.deps.onDisposed?.();\n      }\n      this.frames = [];\n      this.persistedByRef.clear();\n      this.observation = null;\n      this.disposed = true;\n    }\n  }\n\n  private async persistUnstoredFrames(): Promise<void> {\n    const uniqueFrames = this.dedupeRefs(\n      this.frames.filter((frame) => !frame.persisted),\n    );\n    const uncachedFrames = uniqueFrames.filter(\n      (frame) => !this.persistedByRef.has(frame.ref),\n    );\n\n    for (\n      let start = 0;\n      start < uncachedFrames.length;\n      start += DECODE_BATCH_SIZE\n    ) {\n      const batch = uncachedFrames.slice(start, start + DECODE_BATCH_SIZE);\n      const decoded = this.source\n        ? await this.source.decode(batch)\n        : batch.map((frame) => {\n            assert(\n              isImageDataUrl(frame.ref),\n              'fallback observation frame must be an image data URL',\n            );\n            return frame.ref;\n          });\n      assert(\n        decoded.length === batch.length,\n        'frame source decode() must return one image per frame handle',\n      );\n      const preparedDataUrls = await Promise.all(\n        decoded.map((dataUrl) => this.prepareFrameForPersistence(dataUrl)),\n      );\n      for (let index = 0; index < batch.length; index++) {\n        this.persistedByRef.set(\n          batch[index].ref,\n          this.writer.persistFrame(\n            preparedDataUrls[index],\n            batch[index].capturedAt,\n          ),\n        );\n      }\n    }\n\n    for (const frame of this.frames) {\n      frame.persisted ??= this.persistedByRef.get(frame.ref);\n    }\n    if (uncachedFrames.length > 0) {\n      debug(`decoded and persisted ${uncachedFrames.length} source frames`);\n    }\n  }\n\n  private async captureOnce(): Promise<void> {\n    try {\n      if (this.source) {\n        const frame = this.source.latest();\n        if (!frame) return;\n        if (isImageDataUrl(frame.ref)) {\n          const dataUrl = await this.prepareFrameForPersistence(frame.ref);\n          const persisted = this.writer.persistFrame(dataUrl, frame.capturedAt);\n          this.pushFrame({\n            ref: persisted.path,\n            capturedAt: frame.capturedAt,\n            persisted,\n          });\n        } else {\n          this.pushFrame(frame);\n        }\n        return;\n      }\n      const dataUrl = await this.prepareFrameForPersistence(\n        await this.deps.screenshot(),\n      );\n      const persisted = this.writer.persistFrame(dataUrl, Date.now());\n      this.pushFrame({\n        ref: persisted.path,\n        capturedAt: persisted.capturedAt,\n        persisted,\n      });\n    } catch (error) {\n      debug(`frame capture failed, skipping tick: ${error}`);\n    }\n  }\n\n  private async prepareFrameForPersistence(dataUrl: string): Promise<string> {\n    let preparedDataUrl = dataUrl;\n    if (this.screenshotShrinkFactor > 1) {\n      const { width, height } = await imageInfoOfBase64(dataUrl);\n      preparedDataUrl = await resizeImgBase64(dataUrl, {\n        width: Math.round(width / this.screenshotShrinkFactor),\n        height: Math.round(height / this.screenshotShrinkFactor),\n      });\n    }\n\n    return convertPngBase64ToJpeg(preparedDataUrl, OBSERVATION_JPEG_QUALITY);\n  }\n\n  private async runLoop(): Promise<void> {\n    while (!this.stopped) {\n      const tickStart = Date.now();\n      await this.captureOnce();\n      while (!this.stopped && Date.now() - tickStart < this.intervalMs) {\n        await new Promise((resolve) => setTimeout(resolve, 50));\n      }\n    }\n  }\n\n  private pushFrame(frame: DeviceFrameRef | BufferedFrame): void {\n    this.frames.push(frame);\n    if (this.frames.length > this.maxFrames) {\n      this.frames = this.thinBuffer(this.frames);\n      this.writer.pruneFrames(\n        this.frames.flatMap((retainedFrame) =>\n          retainedFrame.persisted ? [retainedFrame.persisted] : [],\n        ),\n      );\n      debug(`frame buffer thinned to ${this.frames.length} frames`);\n    }\n  }\n\n  private thinBuffer(frames: BufferedFrame[]): BufferedFrame[] {\n    if (frames.length <= 1) return frames;\n    const isChangePoint = new Array(frames.length).fill(false);\n    isChangePoint[0] = true;\n    for (let index = 1; index < frames.length; index++) {\n      if (frames[index].ref !== frames[index - 1].ref) {\n        isChangePoint[index] = true;\n      }\n    }\n    isChangePoint[frames.length - 1] = true;\n\n    let result: BufferedFrame[] = [];\n    let staticCounter = 0;\n    for (let index = 0; index < frames.length; index++) {\n      if (isChangePoint[index]) {\n        result.push(frames[index]);\n        staticCounter = 0;\n      } else if (staticCounter % 2 === 0) {\n        result.push(frames[index]);\n        staticCounter++;\n      } else {\n        staticCounter++;\n      }\n    }\n\n    if (result.length > this.maxFrames) {\n      const step = result.length / this.maxFrames;\n      const sampled: BufferedFrame[] = [];\n      for (let index = 0; index < this.maxFrames; index++) {\n        sampled.push(result[Math.floor(index * step)]);\n      }\n      sampled[this.maxFrames - 1] = result[result.length - 1];\n      result = sampled;\n    }\n    return result;\n  }\n\n  private dedupeRefs(frames: BufferedFrame[]): BufferedFrame[] {\n    const seen = new Set<unknown>();\n    const result: BufferedFrame[] = [];\n    for (const frame of frames) {\n      if (!seen.has(frame.ref)) {\n        seen.add(frame.ref);\n        result.push(frame);\n      }\n    }\n    return result;\n  }\n}\n\n/** Rebuild model-facing temporal context from resolved image file paths. */\nexport function uiContextFromObservationRecord(\n  record: UIObservationRecord,\n): UIContext {\n  assert(\n    record.type === 'midscene_ui_observation' && record.version === 1,\n    'invalid UI observation record type or version',\n  );\n  assert(record.frames.length > 0, 'UI observation record contains no frames');\n  assert(\n    Number.isFinite(record.shotSize.width) && record.shotSize.width > 0,\n    'UI observation record shot width must be positive',\n  );\n  assert(\n    Number.isFinite(record.shotSize.height) && record.shotSize.height > 0,\n    'UI observation record shot height must be positive',\n  );\n  assert(\n    Number.isFinite(record.shrunkShotToLogicalRatio) &&\n      record.shrunkShotToLogicalRatio > 0,\n    'UI observation record screenshot ratio must be positive',\n  );\n  const screenshotSequence = record.frames.map((frame) =>\n    ScreenshotItem.fromFile(frame.path, frame.mimeType, frame.capturedAt),\n  );\n  return {\n    screenshot: screenshotSequence[screenshotSequence.length - 1],\n    screenshotSequence,\n    shotSize: { ...record.shotSize },\n    shrunkShotToLogicalRatio: record.shrunkShotToLogicalRatio,\n  };\n}\n"],"names":["debug","getDebug","warnObserver","DEFAULT_INTERVAL_MS","MIN_INTERVAL_MS","DEFAULT_MAX_FRAMES","FIRST_FRAME_TIMEOUT_MS","DEFAULT_WATCHDOG_MS","MAX_FRAMES_PER_RECORD","DECODE_BATCH_SIZE","OBSERVATION_JPEG_QUALITY","UIObservationImpl","assert","options","undefined","demand","prompt","assertion","message","cloneUIObservationRecord","record","insight","disposeRecord","onDisposed","isImageDataUrl","value","UIObserverImpl","Date","error","waitStart","Promise","resolve","setTimeout","clearTimeout","representativePromise","representative","lastFrame","ScreenshotItem","endedAt","sampledFrames","frame","frames","metadata","uniqueFrames","uncachedFrames","start","batch","decoded","preparedDataUrls","dataUrl","index","persisted","preparedDataUrl","width","height","imageInfoOfBase64","resizeImgBase64","Math","convertPngBase64ToJpeg","tickStart","retainedFrame","isChangePoint","Array","result","staticCounter","step","sampled","seen","Set","deps","opt","Map","UIObservationRecordWriter","uiContextFromObservationRecord","Number","screenshotSequence"],"mappings":";;;;;;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,eAAeD,SAAS,eAAe;IAAE,SAAS;AAAK;AAE7D,MAAME,sBAAsB;AAC5B,MAAMC,kBAAkB;AACxB,MAAMC,qBAAqB;AAC3B,MAAMC,yBAAyB;AAC/B,MAAMC,sBAAsB;AAC5B,MAAMC,wBAAwB;AAC9B,MAAMC,oBAAoB;AAC1B,MAAMC,2BAA2B;AAyC1B,MAAMC;IAaX,IAAI,aAAqB;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM;IAClC;IAEA,IAAI,YAAoB;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS;IAC9B;IAEA,IAAI,UAAkB;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO;IAC5B;IAEQ,eAAqB;QAC3BC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE;IACzB;IAEQ,yBAAyBC,OAAmC,EAAQ;QAC1ED,OACEC,SAAS,gBAAgBC,QACzB;IAEJ;IAEA,MAAM,QACJC,MAA2B,EAC3BF,OAAiC,EACZ;QACrB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAaE,QAAQF;IAClD;IAEA,MAAM,UACJG,MAAmB,EACnBH,OAAiC,EACf;QAClB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAACG,QAAQH;IACxC;IAEA,MAAM,SACJG,MAAmB,EACnBH,OAAiC,EAChB;QACjB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACG,QAAQH;IACvC;IAEA,MAAM,SACJG,MAAmB,EACnBH,OAAiC,EAChB;QACjB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACG,QAAQH;IACvC;IAEA,MAAM,MACJG,MAAmB,EACnBH,OAAiC,EAChB;QACjB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAACG,QAAQH;IACpC;IAEA,MAAM,SACJI,SAAsB,EACtBC,OAAgB,EAChBL,OAAkC,EACM;QACxC,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACI,WAAWC,SAASL;IACnD;IAGA,MAAM,eAA6C;QACjD,IAAI,CAAC,YAAY;QACjB,OAAOM,yBAAyB,IAAI,CAAC,MAAM;IAC7C;IAEA,MAAM,UAAyB;QAC7B,IAAI,IAAI,CAAC,QAAQ,EAAE;QACnB,IAAI,CAAC,aAAa;QAClB,IAAI,CAAC,QAAQ,GAAG;QAChB,IAAI,CAAC,UAAU;IACjB;IAlGA,YACEC,MAA2B,EACVC,OAAmB,EACnBC,aAA0B,EAC1BC,UAAuB,CACxC;;;;QARF,uBAAQ,YAAR;QACA,uBAAiB,UAAjB;aAImBF,OAAO,GAAPA;aACAC,aAAa,GAAbA;aACAC,UAAU,GAAVA;aAPX,QAAQ,GAAG;QASjB,IAAI,CAAC,MAAM,GAAGJ,yBAAyBC;IACzC;AA4FF;AAOA,SAASI,eAAeC,KAAc;IACpC,OACE,AAAiB,YAAjB,OAAOA,SACP,sCAAsC,IAAI,CAACA;AAE/C;AAKO,MAAMC;IAoCX,IAAI,qBAA6B;QAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM;IAC3B;IAEA,MAAM,QAAuB;QAC3Bd,OAAO,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QAC3C,IAAI,CAAC,SAAS,GAAGe,KAAK,GAAG;QACzB,IAAI;YACF,IAAI,CAAC,MAAM,GAAI,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,MAAO;QACvD,EAAE,OAAOC,OAAO;YACd5B,MAAM,CAAC,qDAAqD,EAAE4B,OAAO;YACrE,IAAI,CAAC,MAAM,GAAG;QAChB;QACA,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,MAAM;QACjC,IAAI,IAAI,CAAC,aAAa,EACpB5B,MAAM;aACD;YACL,MAAM6B,YAAYF,KAAK,GAAG;YAC1B,MACE,CAAC,IAAI,CAAC,MAAM,CAAE,MAAM,MACpBA,KAAK,GAAG,KAAKE,YAAYvB,uBAEzB,MAAM,IAAIwB,QAAQ,CAACC,UAAYC,WAAWD,SAAS;YAErD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAE,MAAM,IACtB/B,MACE,CAAC,sBAAsB,EAAEM,uBAAuB,mBAAmB,CAAC;QAG1E;QACA,MAAM,IAAI,CAAC,WAAW;QACtB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO;QAE/B,IAAI,IAAI,CAAC,UAAU,GAAG,GAAG;YACvB,IAAI,CAAC,aAAa,GAAG0B,WAAW;gBAC9B9B,aACE,CAAC,8BAA8B,EAAE,IAAI,CAAC,UAAU,CAAC,kDAAkD,CAAC;gBAEtG,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,KAAO;YAC3B,GAAG,IAAI,CAAC,UAAU;YAClB,IACE,AACA,cADA,OAAQ,IAAI,CAAC,aAAa,CAA4B,KAAK,EAG1D,IAAI,CAAC,aAAa,CAA2B,KAAK;QAEvD;IACF;IAEA,OAAmC;QACjC,IAAI,CAAC,IAAI,CAAC,WAAW,EACnB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY;QAEtC,OAAO,IAAI,CAAC,WAAW;IACzB;IAEA,MAAc,eAA2C;QACvD,IAAI,CAAC,OAAO,GAAG;QACf,IAAI,IAAI,CAAC,aAAa,EAAE;YACtB+B,aAAa,IAAI,CAAC,aAAa;YAC/B,IAAI,CAAC,aAAa,GAAG;QACvB;QACA,MAAM,IAAI,CAAC,WAAW;QAEtB,IAAI;YACF,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,GACvB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC,CAACL;gBACxD5B,MAAM,CAAC,oDAAoD,EAAE4B,OAAO;YACtE;YAEF,MAAMM,wBAAwB,IAAI,CAAC,IAAI,CAAC,qBAAqB;YAC7D,MAAM,GAAGC,eAAe,GAAG,MAAML,QAAQ,GAAG,CAAC;gBAC3C,IAAI,CAAC,cAAc;gBACnBI;aACD;YAED,MAAME,YAAY,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,MAAM,IAAIA,WAAW,WAAW;gBACvC,IAAI,CAAC,mBAAmB,GAAG;oBACzB,GAAGA,UAAU,SAAS;oBACtB,YAAYA,UAAU,UAAU;gBAClC;gBACAD,eAAe,UAAU,GAAGE,eAAe,QAAQ,CACjD,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAACD,UAAU,SAAS,GAChDA,UAAU,SAAS,CAAC,QAAQ,EAC5BA,UAAU,UAAU;gBAEtBpC,MAAM;YACR;YACA,IAAI,CAAC,cAAc,GAAGmC;YACtB,MAAMG,UAAUX,KAAK,GAAG;YACxB,MAAMP,SAAS,MAAM,IAAI,CAAC,cAAc,CAACkB;YACzC,IAAI,CAAC,WAAW,GAAG,IAAI3B,kBACrBS,QACA,IAAI,CAAC,IAAI,CAAC,aAAa,CAACA,SACxB,IAAM,IAAI,CAAC,MAAM,CAAC,OAAO,IACzB,IAAI,CAAC,IAAI,CAAC,UAAU;YAEtB,OAAO,IAAI,CAAC,WAAW;QACzB,SAAU;YACR,IAAI,IAAI,CAAC,MAAM,EACb,IAAI;gBACF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI;YACxB,EAAE,OAAOQ,OAAO;gBACd5B,MAAM,CAAC,6BAA6B,EAAE4B,OAAO;YAC/C;YAEF5B,MACE,CAAC,yBAAyB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,oCAAoC,CAAC;YAEtF,IAAI,CAAC,IAAI,CAAC,SAAS;QACrB;IACF;IAEA,MAAc,eAAesC,OAAe,EAAgC;QAC1E1B,OACE,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,EACnC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,MAAM,IAAI,CAAC,cAAc;YACzB,IAAI,CAAC,cAAc,GAAG;QACxB;QACA,MAAM,IAAI,CAAC,qBAAqB;QAEhC,MAAM2B,gBAAgB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAACC;YACrC5B,OAAO4B,MAAM,SAAS,EAAE;YACxB,OAAO;gBAAE,GAAGA,MAAM,SAAS;gBAAE,YAAYA,MAAM,UAAU;YAAC;QAC5D;QAEA,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,MAAML,iBAAiB,IAAI,CAAC,cAAc;YAC1C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CACjD,MAAM,IAAI,CAAC,0BAA0B,CAACA,eAAe,UAAU,CAAC,MAAM,GACtEA,eAAe,UAAU,CAAC,UAAU;YAEtCA,eAAe,UAAU,GAAGE,eAAe,QAAQ,CACjD,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,mBAAmB,GACrD,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EACjC,IAAI,CAAC,mBAAmB,CAAC,UAAU;QAEvC;QAEA,MAAMI,SAAS;eAAIF;YAAe,IAAI,CAAC,mBAAmB;SAAC;QAC3D,IAAIE,OAAO,MAAM,GAAGjC,uBAClBN,aACE,CAAC,mBAAmB,EAAEuC,OAAO,MAAM,CAAC,oBAAoB,EAAEjC,sBAAsB,wJAAwJ,CAAC;QAG7OR,MACE,CAAC,UAAU,EAAEyC,OAAO,MAAM,CAAC,iCAAiC,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAE/G,MAAMC,WAAwC;YAC5C,WAAW,IAAI,CAAC,SAAS;YACzBJ;YACA,UAAU;gBAAE,GAAG,IAAI,CAAC,cAAc,CAAE,QAAQ;YAAC;YAC7C,0BAA0B,IAAI,CAAC,cAAc,CAAE,wBAAwB;QACzE;QACA,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAACG,QAAQC;IACtC;IAGA,MAAM,UAAyB;QAC7B,IAAI,IAAI,CAAC,QAAQ,EAAE;QACnB,IAAI;YACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW;iBACjB,IAAI,CAAC,IAAI,CAAC,OAAO,EACtB,MAAM,IAAI,CAAC,IAAI;QAEnB,SAAU;YACR,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO;iBACzB;gBACL,IAAI,CAAC,MAAM,CAAC,OAAO;gBACnB,IAAI,CAAC,IAAI,CAAC,UAAU;YACtB;YACA,IAAI,CAAC,MAAM,GAAG,EAAE;YAChB,IAAI,CAAC,cAAc,CAAC,KAAK;YACzB,IAAI,CAAC,WAAW,GAAG;YACnB,IAAI,CAAC,QAAQ,GAAG;QAClB;IACF;IAEA,MAAc,wBAAuC;QACnD,MAAMC,eAAe,IAAI,CAAC,UAAU,CAClC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAACH,QAAU,CAACA,MAAM,SAAS;QAEhD,MAAMI,iBAAiBD,aAAa,MAAM,CACxC,CAACH,QAAU,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAACA,MAAM,GAAG;QAG/C,IACE,IAAIK,QAAQ,GACZA,QAAQD,eAAe,MAAM,EAC7BC,SAASpC,kBACT;YACA,MAAMqC,QAAQF,eAAe,KAAK,CAACC,OAAOA,QAAQpC;YAClD,MAAMsC,UAAU,IAAI,CAAC,MAAM,GACvB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAACD,SACzBA,MAAM,GAAG,CAAC,CAACN;gBACT5B,OACEY,eAAegB,MAAM,GAAG,GACxB;gBAEF,OAAOA,MAAM,GAAG;YAClB;YACJ5B,OACEmC,QAAQ,MAAM,KAAKD,MAAM,MAAM,EAC/B;YAEF,MAAME,mBAAmB,MAAMlB,QAAQ,GAAG,CACxCiB,QAAQ,GAAG,CAAC,CAACE,UAAY,IAAI,CAAC,0BAA0B,CAACA;YAE3D,IAAK,IAAIC,QAAQ,GAAGA,QAAQJ,MAAM,MAAM,EAAEI,QACxC,IAAI,CAAC,cAAc,CAAC,GAAG,CACrBJ,KAAK,CAACI,MAAM,CAAC,GAAG,EAChB,IAAI,CAAC,MAAM,CAAC,YAAY,CACtBF,gBAAgB,CAACE,MAAM,EACvBJ,KAAK,CAACI,MAAM,CAAC,UAAU;QAI/B;QAEA,KAAK,MAAMV,SAAS,IAAI,CAAC,MAAM,CAC7BA,MAAM,SAAS,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,CAACA,MAAM,GAAG;QAEvD,IAAII,eAAe,MAAM,GAAG,GAC1B5C,MAAM,CAAC,sBAAsB,EAAE4C,eAAe,MAAM,CAAC,cAAc,CAAC;IAExE;IAEA,MAAc,cAA6B;QACzC,IAAI;YACF,IAAI,IAAI,CAAC,MAAM,EAAE;gBACf,MAAMJ,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM;gBAChC,IAAI,CAACA,OAAO;gBACZ,IAAIhB,eAAegB,MAAM,GAAG,GAAG;oBAC7B,MAAMS,UAAU,MAAM,IAAI,CAAC,0BAA0B,CAACT,MAAM,GAAG;oBAC/D,MAAMW,YAAY,IAAI,CAAC,MAAM,CAAC,YAAY,CAACF,SAAST,MAAM,UAAU;oBACpE,IAAI,CAAC,SAAS,CAAC;wBACb,KAAKW,UAAU,IAAI;wBACnB,YAAYX,MAAM,UAAU;wBAC5BW;oBACF;gBACF,OACE,IAAI,CAAC,SAAS,CAACX;gBAEjB;YACF;YACA,MAAMS,UAAU,MAAM,IAAI,CAAC,0BAA0B,CACnD,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU;YAE5B,MAAME,YAAY,IAAI,CAAC,MAAM,CAAC,YAAY,CAACF,SAAStB,KAAK,GAAG;YAC5D,IAAI,CAAC,SAAS,CAAC;gBACb,KAAKwB,UAAU,IAAI;gBACnB,YAAYA,UAAU,UAAU;gBAChCA;YACF;QACF,EAAE,OAAOvB,OAAO;YACd5B,MAAM,CAAC,qCAAqC,EAAE4B,OAAO;QACvD;IACF;IAEA,MAAc,2BAA2BqB,OAAe,EAAmB;QACzE,IAAIG,kBAAkBH;QACtB,IAAI,IAAI,CAAC,sBAAsB,GAAG,GAAG;YACnC,MAAM,EAAEI,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMC,kBAAkBN;YAClDG,kBAAkB,MAAMI,gBAAgBP,SAAS;gBAC/C,OAAOQ,KAAK,KAAK,CAACJ,QAAQ,IAAI,CAAC,sBAAsB;gBACrD,QAAQI,KAAK,KAAK,CAACH,SAAS,IAAI,CAAC,sBAAsB;YACzD;QACF;QAEA,OAAOI,uBAAuBN,iBAAiB1C;IACjD;IAEA,MAAc,UAAyB;QACrC,MAAO,CAAC,IAAI,CAAC,OAAO,CAAE;YACpB,MAAMiD,YAAYhC,KAAK,GAAG;YAC1B,MAAM,IAAI,CAAC,WAAW;YACtB,MAAO,CAAC,IAAI,CAAC,OAAO,IAAIA,KAAK,GAAG,KAAKgC,YAAY,IAAI,CAAC,UAAU,CAC9D,MAAM,IAAI7B,QAAQ,CAACC,UAAYC,WAAWD,SAAS;QAEvD;IACF;IAEQ,UAAUS,KAAqC,EAAQ;QAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CAACA;QACjB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;YACvC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM;YACzC,IAAI,CAAC,MAAM,CAAC,WAAW,CACrB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAACoB,gBACnBA,cAAc,SAAS,GAAG;oBAACA,cAAc,SAAS;iBAAC,GAAG,EAAE;YAG5D5D,MAAM,CAAC,wBAAwB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;QAC9D;IACF;IAEQ,WAAWyC,MAAuB,EAAmB;QAC3D,IAAIA,OAAO,MAAM,IAAI,GAAG,OAAOA;QAC/B,MAAMoB,gBAAgB,IAAIC,MAAMrB,OAAO,MAAM,EAAE,IAAI,CAAC;QACpDoB,aAAa,CAAC,EAAE,GAAG;QACnB,IAAK,IAAIX,QAAQ,GAAGA,QAAQT,OAAO,MAAM,EAAES,QACzC,IAAIT,MAAM,CAACS,MAAM,CAAC,GAAG,KAAKT,MAAM,CAACS,QAAQ,EAAE,CAAC,GAAG,EAC7CW,aAAa,CAACX,MAAM,GAAG;QAG3BW,aAAa,CAACpB,OAAO,MAAM,GAAG,EAAE,GAAG;QAEnC,IAAIsB,SAA0B,EAAE;QAChC,IAAIC,gBAAgB;QACpB,IAAK,IAAId,QAAQ,GAAGA,QAAQT,OAAO,MAAM,EAAES,QACzC,IAAIW,aAAa,CAACX,MAAM,EAAE;YACxBa,OAAO,IAAI,CAACtB,MAAM,CAACS,MAAM;YACzBc,gBAAgB;QAClB,OAAO,IAAIA,gBAAgB,MAAM,GAAG;YAClCD,OAAO,IAAI,CAACtB,MAAM,CAACS,MAAM;YACzBc;QACF,OACEA;QAIJ,IAAID,OAAO,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;YAClC,MAAME,OAAOF,OAAO,MAAM,GAAG,IAAI,CAAC,SAAS;YAC3C,MAAMG,UAA2B,EAAE;YACnC,IAAK,IAAIhB,QAAQ,GAAGA,QAAQ,IAAI,CAAC,SAAS,EAAEA,QAC1CgB,QAAQ,IAAI,CAACH,MAAM,CAACN,KAAK,KAAK,CAACP,QAAQe,MAAM;YAE/CC,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,EAAE,GAAGH,MAAM,CAACA,OAAO,MAAM,GAAG,EAAE;YACvDA,SAASG;QACX;QACA,OAAOH;IACT;IAEQ,WAAWtB,MAAuB,EAAmB;QAC3D,MAAM0B,OAAO,IAAIC;QACjB,MAAML,SAA0B,EAAE;QAClC,KAAK,MAAMvB,SAASC,OAClB,IAAI,CAAC0B,KAAK,GAAG,CAAC3B,MAAM,GAAG,GAAG;YACxB2B,KAAK,GAAG,CAAC3B,MAAM,GAAG;YAClBuB,OAAO,IAAI,CAACvB;QACd;QAEF,OAAOuB;IACT;IA3WA,YACmBM,IAAoB,EACrCC,GAAsB,CACtB;;QAvBF,uBAAQ,UAAR;QACA,uBAAQ,UAAR;QACA,uBAAQ,iBAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,YAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,kBAAR;QACA,uBAAQ,uBAAR;QACA,uBAAQ,iBAAR;QACA,uBAAQ,kBAAR;QACA,uBAAQ,kBAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,aAAR;QACA,uBAAiB,cAAjB;QACA,uBAAiB,aAAjB;QACA,uBAAiB,cAAjB;QACA,uBAAiB,0BAAjB;QACA,uBAAiB,UAAjB;aAGmBD,IAAI,GAAJA;aArBX,MAAM,GAAoB,EAAE;aAC5B,MAAM,GAA6B;aACnC,aAAa,GAAG;aAChB,OAAO,GAAG;aACV,QAAQ,GAAG;aACX,WAAW,GAAyB;aACpC,WAAW,GAAsC;aACjD,cAAc,GAAqB;aACnC,mBAAmB,GAA8B;aACjD,aAAa,GAAyC;aACtD,cAAc,GAAyB;aACvC,cAAc,GAAG,IAAIE;aACrB,WAAW,GAA6B;aACxC,SAAS,GAAG;QAWlB,IAAI,CAAC,UAAU,GAAGd,KAAK,GAAG,CACxBrD,iBACAkE,KAAK,cAAcnE;QAErB,IAAI,CAAC,SAAS,GAAGsD,KAAK,GAAG,CAAC,GAAGa,KAAK,aAAajE;QAC/C,IAAI,CAAC,UAAU,GAAGiE,KAAK,cAAc/D;QACrC,IAAI,CAAC,sBAAsB,GAAG8D,KAAK,sBAAsB,IAAI;QAC7D,IAAI,CAAC,MAAM,GACTA,KAAK,uBAAuB,IAAI,IAAIG;IACxC;AA+VF;AAGO,SAASC,+BACdrD,MAA2B;IAE3BR,OACEQ,AAAgB,8BAAhBA,OAAO,IAAI,IAAkCA,AAAmB,MAAnBA,OAAO,OAAO,EAC3D;IAEFR,OAAOQ,OAAO,MAAM,CAAC,MAAM,GAAG,GAAG;IACjCR,OACE8D,OAAO,QAAQ,CAACtD,OAAO,QAAQ,CAAC,KAAK,KAAKA,OAAO,QAAQ,CAAC,KAAK,GAAG,GAClE;IAEFR,OACE8D,OAAO,QAAQ,CAACtD,OAAO,QAAQ,CAAC,MAAM,KAAKA,OAAO,QAAQ,CAAC,MAAM,GAAG,GACpE;IAEFR,OACE8D,OAAO,QAAQ,CAACtD,OAAO,wBAAwB,KAC7CA,OAAO,wBAAwB,GAAG,GACpC;IAEF,MAAMuD,qBAAqBvD,OAAO,MAAM,CAAC,GAAG,CAAC,CAACoB,QAC5CH,eAAe,QAAQ,CAACG,MAAM,IAAI,EAAEA,MAAM,QAAQ,EAAEA,MAAM,UAAU;IAEtE,OAAO;QACL,YAAYmC,kBAAkB,CAACA,mBAAmB,MAAM,GAAG,EAAE;QAC7DA;QACA,UAAU;YAAE,GAAGvD,OAAO,QAAQ;QAAC;QAC/B,0BAA0BA,OAAO,wBAAwB;IAC3D;AACF"}

package/dist/es/agent/utils.mjs

@@ -5,7 +5,7 @@ import { ScreenshotItem } from "../screenshot-item.mjs";
 import { uploadTestInfoToServer } from "../utils.mjs";
 import { MIDSCENE_REPORT_QUIET, MIDSCENE_REPORT_TAG_NAME, globalConfigManager } from "@midscene/shared/env";
 import { generateElementByRect } from "@midscene/shared/extractor";
-import { convertImgBufferToJpeg, createImgBase64ByFormat, imageInfoOfBase64, parseBase64, resizeImgBase64 } from "@midscene/shared/img";
+import { convertPngBase64ToJpeg, createImgBase64ByFormat, imageInfoOfBase64, resizeImgBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert, ifInBrowser, logMsg, uuid } from "@midscene/shared/utils";
 import dayjs from "dayjs";
@@ -94,12 +94,7 @@ async function commonContextParser(interfaceInstance, _opt) {
         };
     }
     {
-        let outputScreenshotBase64 = screenshotBase64;
-        const { mimeType, body } = parseBase64(screenshotBase64);
-        if ('image/png' === mimeType.toLowerCase()) {
-            const jpegBuffer = await convertImgBufferToJpeg(Buffer.from(body, 'base64'), 90);
-            outputScreenshotBase64 = createImgBase64ByFormat('jpeg', jpegBuffer.toString('base64'));
-        }
+        const outputScreenshotBase64 = await convertPngBase64ToJpeg(screenshotBase64, 90);
         return {
             shotSize: {
                 width: imgWidth,
@@ -181,7 +176,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.11.0";
+const getMidsceneVersion = ()=>"1.11.1-beta-20260818123028.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/agent/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\nimport { pixelBboxToRect } from '@/ai-model/workflows/grounding/locate-result-rect';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PixelBbox,\n  PlanningLocateParam,\n  PlanningLocateParamWithLocatedPixelBbox,\n  Rect,\n  ScrollParam,\n  Size,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport {\n  convertImgBufferToJpeg,\n  createImgBase64ByFormat,\n  imageInfoOfBase64,\n  parseBase64,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, ifInBrowser, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst agentDebug = getDebug('agent');\nconst screenshotDataUrlPattern = /^data:image\\/[a-zA-Z0-9.+-]+;base64,/i;\n\nconst inferBase64ImageFormat = (base64Body: string) => {\n  if (base64Body.startsWith('iVBORw0KGgo')) {\n    return 'png';\n  }\n  return 'jpeg';\n};\n\nconst normalizeScreenshotBase64 = (screenshotBase64: string) => {\n  const trimmedBase64 = screenshotBase64.trim();\n  if (screenshotDataUrlPattern.test(trimmedBase64)) {\n    return trimmedBase64;\n  }\n\n  const base64Body = trimmedBase64.replace(/\\s/g, '');\n  assert(base64Body, 'screenshotBase64 must include image data');\n  return createImgBase64ByFormat(\n    inferBase64ImageFormat(base64Body),\n    base64Body,\n  );\n};\n\nconst legacyScrollTypeMap = {\n  once: 'singleAction',\n  untilBottom: 'scrollToBottom',\n  untilTop: 'scrollToTop',\n  untilRight: 'scrollToRight',\n  untilLeft: 'scrollToLeft',\n} as const;\n\nexport const normalizeScrollType = (\n  scrollType: string | undefined,\n): ScrollParam['scrollType'] | undefined => {\n  if (!scrollType) {\n    return undefined;\n  }\n\n  if (scrollType in legacyScrollTypeMap) {\n    return legacyScrollTypeMap[scrollType as keyof typeof legacyScrollTypeMap];\n  }\n\n  return scrollType as ScrollParam['scrollType'];\n};\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  } else {\n    // For screenshots that do not need shrinking, convert PNG to JPEG to reduce the image payload in model requests and reports. (Shrunk images are already JPEG.)\n    // This mainly covers Android's default screenshot path, which produces PNG screenshots.\n    // Compared with conversion on Android, centralizing it here means each platform does not need to handle screenshot formats itself, and allows future output formats such as WebP.\n    // Built-in paths that already output JPEG are unaffected, and custom devices that output JPEG will not be compressed again.\n    // The Web platform already outputs JPEG, so it does not enter this branch. Other built-in device platforms run in Node, where Sharp conversion is fast enough that its extra cost is negligible.\n    let outputScreenshotBase64 = screenshotBase64;\n    const { mimeType, body } = parseBase64(screenshotBase64);\n    if (mimeType.toLowerCase() === 'image/png') {\n      const jpegBuffer = await convertImgBufferToJpeg(\n        Buffer.from(body, 'base64'),\n        90,\n      );\n      outputScreenshotBase64 = createImgBase64ByFormat(\n        'jpeg',\n        jpegBuffer.toString('base64'),\n      );\n    }\n\n    return {\n      shotSize: {\n        width: imgWidth,\n        height: imgHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(\n        outputScreenshotBase64,\n        screenshotCapturedAt,\n      ),\n      shrunkShotToLogicalRatio,\n    };\n  }\n}\n\nexport async function createScreenshotBoundUIContext(\n  screenshotBase64: string,\n  opt: {\n    screenshotSize?: Size;\n  },\n): Promise<UIContext> {\n  const normalizedScreenshotBase64 =\n    normalizeScreenshotBase64(screenshotBase64);\n  const actualScreenshotSize = await imageInfoOfBase64(\n    normalizedScreenshotBase64,\n  );\n  if (\n    opt.screenshotSize &&\n    (opt.screenshotSize.width !== actualScreenshotSize.width ||\n      opt.screenshotSize.height !== actualScreenshotSize.height)\n  ) {\n    agentDebug(\n      'describeElementAtPoint screenshotSize mismatch, use actual size',\n      {\n        provided: opt.screenshotSize,\n        actual: actualScreenshotSize,\n      },\n    );\n  }\n\n  return {\n    screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n    shotSize: actualScreenshotSize,\n    shrunkShotToLogicalRatio: 1,\n    _isFrozen: true,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\ntype NormalizeFilePathsOptions = {\n  fileExists?: (path: string) => boolean;\n  isInBrowser?: boolean;\n  resolvePath?: (path: string) => string;\n  wslDistroName?: string;\n  cwd?: string;\n};\n\nexport function normalizeFilePaths(\n  files: string[],\n  options: NormalizeFilePathsOptions = {},\n): string[] {\n  const {\n    fileExists = existsSync,\n    isInBrowser = ifInBrowser,\n    resolvePath = resolve,\n    wslDistroName = process.env.WSL_DISTRO_NAME,\n    cwd = process.cwd(),\n  } = options;\n\n  if (isInBrowser) {\n    throw new Error('File chooser is not supported in browser environment');\n  }\n\n  return files.map((file) => {\n    const absolutePath = resolvePath(file);\n    if (!fileExists(absolutePath)) {\n      throw new Error(\n        `File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`,\n      );\n    }\n\n    if (!wslDistroName) {\n      return absolutePath;\n    }\n\n    const wslMount = absolutePath.match(/^\\/mnt\\/([a-z])\\//);\n    if (wslMount) {\n      return `${wslMount[1].toUpperCase()}:\\\\${absolutePath.slice(7).replace(/\\//g, '\\\\')}`;\n    }\n\n    return `\\\\\\\\wsl$\\\\${wslDistroName}${absolutePath.replace(/\\//g, '\\\\')}`;\n  });\n}\n\nexport function isPixelBbox(value: unknown): value is PixelBbox {\n  return (\n    Array.isArray(value) &&\n    value.length === 4 &&\n    value.every((item) => typeof item === 'number' && Number.isFinite(item))\n  );\n}\n\ntype PlanningLocateParamWithMaybeLocatedPixelBbox = PlanningLocateParam & {\n  locatedPixelBbox?: unknown;\n};\n\nexport function ifPlanLocateParamHasLocatedPixelBbox(\n  planLocateParam: PlanningLocateParamWithMaybeLocatedPixelBbox,\n): planLocateParam is PlanningLocateParamWithLocatedPixelBbox {\n  return isPixelBbox(planLocateParam.locatedPixelBbox);\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParamWithLocatedPixelBbox,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  const rect = pixelBboxToRect(planLocateParam.locatedPixelBbox);\n\n  const element = generateElementByRect(\n    rect,\n    typeof planLocateParam.prompt === 'string'\n      ? planLocateParam.prompt\n      : planLocateParam.prompt?.prompt || '',\n  );\n  return element;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["agentDebug","getDebug","screenshotDataUrlPattern","inferBase64ImageFormat","base64Body","normalizeScreenshotBase64","screenshotBase64","trimmedBase64","assert","createImgBase64ByFormat","legacyScrollTypeMap","normalizeScrollType","scrollType","commonContextParser","interfaceInstance","_opt","debug","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","outputScreenshotBase64","mimeType","body","parseBase64","jpegBuffer","convertImgBufferToJpeg","Buffer","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","actualScreenshotSize","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","normalizeFilePaths","files","options","fileExists","existsSync","isInBrowser","ifInBrowser","resolvePath","resolve","wslDistroName","process","cwd","file","absolutePath","wslMount","isPixelBbox","value","Array","item","ifPlanLocateParamHasLocatedPixelBbox","planLocateParam","matchElementFromPlan","rect","pixelBboxToRect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;;;;;;;;;;AAsCA,MAAMA,aAAaC,SAAS;AAC5B,MAAMC,2BAA2B;AAEjC,MAAMC,yBAAyB,CAACC;IAC9B,IAAIA,WAAW,UAAU,CAAC,gBACxB,OAAO;IAET,OAAO;AACT;AAEA,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,gBAAgBD,iBAAiB,IAAI;IAC3C,IAAIJ,yBAAyB,IAAI,CAACK,gBAChC,OAAOA;IAGT,MAAMH,aAAaG,cAAc,OAAO,CAAC,OAAO;IAChDC,OAAOJ,YAAY;IACnB,OAAOK,wBACLN,uBAAuBC,aACvBA;AAEJ;AAEA,MAAMM,sBAAsB;IAC1B,MAAM;IACN,aAAa;IACb,UAAU;IACV,YAAY;IACZ,WAAW;AACb;AAEO,MAAMC,sBAAsB,CACjCC;IAEA,IAAI,CAACA,YACH;IAGF,IAAIA,cAAcF,qBAChB,OAAOA,mBAAmB,CAACE,WAA+C;IAG5E,OAAOA;AACT;AAEO,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQf,SAAS;IAEvBO,OAAOM,mBAAmB;IAE1BE,MAAM;IACN,MAAMC,cAAcH,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNE,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMG,gBAAgB,MAAML,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOM,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIL,MAAM,CAAC,MAAM,EAAEI,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMf,mBAAmB,MAAMQ,kBAAkB,gBAAgB;IACjE,MAAMW,uBAAuBC,KAAK,GAAG;IACrClB,OAAOF,kBAAmB;IAG1BU,MAAM;IACN,MAAM,EAAE,OAAOW,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,kBAAkBvB;IAE1B,IAAI,CAACiB,OAAO,QAAQ,CAACI,aAAa,CAACJ,OAAO,QAAQ,CAACK,YACjD,MAAM,IAAIJ,MACR,CAAC,wFAAwF,EAAEG,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIJ,MACR,CAAC,0FAA0F,EAAEG,SAAS,UAAU,EAAEC,WAAW;IAGjIZ,MAAM,yBAAyBW,UAAU,KAAKC;IAM9C,MAAME,oBAAoBV,eAAeC;IACzC,MAAMU,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBZ;IAExB,IAAIU,sBAAsBC,sBAAsB;QAC9Cf,MACE,CAAC,4CAA4C,EAAEI,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAES,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBX;IAEtB;IAEA,MAAMY,mBAAmBlB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACQ,OAAO,QAAQ,CAACU,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIT,MACR,CAAC,wEAAwE,EAAES,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBhB,MAAM,mBAAmBkB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCjB,MAAM,4BAA4BmB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CjB,MACE,CAAC,mCAAmC,EAAEiB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,gBAAgBlC,kBAAkB;YAC5D,OAAO8B;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,eAAe,MAAM,CAACF,eAAed;YACjDU;QACF;IACF;IAAO;QAML,IAAIO,yBAAyBpC;QAC7B,MAAM,EAAEqC,QAAQ,EAAEC,IAAI,EAAE,GAAGC,YAAYvC;QACvC,IAAIqC,AAA2B,gBAA3BA,SAAS,WAAW,IAAoB;YAC1C,MAAMG,aAAa,MAAMC,uBACvBC,OAAO,IAAI,CAACJ,MAAM,WAClB;YAEFF,yBAAyBjC,wBACvB,QACAqC,WAAW,QAAQ,CAAC;QAExB;QAEA,OAAO;YACL,UAAU;gBACR,OAAOnB;gBACP,QAAQC;YACV;YACA,eAAeM;YACf,YAAYO,eAAe,MAAM,CAC/BC,wBACAjB;YAEFU;QACF;IACF;AACF;AAEO,eAAec,+BACpB3C,gBAAwB,EACxB4C,GAEC;IAED,MAAMC,6BACJ9C,0BAA0BC;IAC5B,MAAM8C,uBAAuB,MAAMvB,kBACjCsB;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKE,qBAAqB,KAAK,IACtDF,IAAI,cAAc,CAAC,MAAM,KAAKE,qBAAqB,MAAK,GAE1DpD,WACE,mEACA;QACE,UAAUkD,IAAI,cAAc;QAC5B,QAAQE;IACV;IAIJ,OAAO;QACL,YAAYX,eAAe,MAAM,CAACU,4BAA4BzB,KAAK,GAAG;QACtE,UAAU0B;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASC,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAUO,SAASG,mBACdC,KAAe,EACfC,UAAqC,CAAC,CAAC;IAEvC,MAAM,EACJC,aAAaC,UAAU,EACvBC,cAAcC,WAAW,EACzBC,cAAcC,OAAO,EACrBC,gBAAgBC,QAAQ,GAAG,CAAC,eAAe,EAC3CC,MAAMD,QAAQ,GAAG,EAAE,EACpB,GAAGR;IAEJ,IAAIG,aACF,MAAM,IAAI/C,MAAM;IAGlB,OAAO2C,MAAM,GAAG,CAAC,CAACW;QAChB,MAAMC,eAAeN,YAAYK;QACjC,IAAI,CAACT,WAAWU,eACd,MAAM,IAAIvD,MACR,CAAC,gBAAgB,EAAEsD,KAAK,eAAe,EAAEC,aAAa,6BAA6B,EAAEF,KAAK;QAI9F,IAAI,CAACF,eACH,OAAOI;QAGT,MAAMC,WAAWD,aAAa,KAAK,CAAC;QACpC,IAAIC,UACF,OAAO,GAAGA,QAAQ,CAAC,EAAE,CAAC,WAAW,GAAG,GAAG,EAAED,aAAa,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,OAAO;QAGvF,OAAO,CAAC,UAAU,EAAEJ,gBAAgBI,aAAa,OAAO,CAAC,OAAO,OAAO;IACzE;AACF;AAEO,SAASE,YAAYC,KAAc;IACxC,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,OAAS,AAAgB,YAAhB,OAAOA,QAAqB7D,OAAO,QAAQ,CAAC6D;AAEtE;AAMO,SAASC,qCACdC,eAA6D;IAE7D,OAAOL,YAAYK,gBAAgB,gBAAgB;AACrD;AAEO,SAASC,qBACdD,eAAwD;IAExD,IAAI,CAACA,iBACH;IAGF,MAAME,OAAOC,gBAAgBH,gBAAgB,gBAAgB;IAE7D,MAAMI,UAAUC,sBACdH,MACA,AAAkC,YAAlC,OAAOF,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;IAExC,OAAOI;AACT;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMT,OACJ,MAAMK,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNrD,KAAK,KAAK,CAACmD,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCnD,KAAK,KAAK,CAACmD,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOO,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,8BAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAvD;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOuD;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNrD,KAAK,KAAK,CAACqD,QAAQ,MAAM,CAAC,EAAE,GAAGvD;YAC/BE,KAAK,KAAK,CAACqD,QAAQ,MAAM,CAAC,EAAE,GAAGvD;SAChC;QACD,MAAM;YACJ,GAAGuD,QAAQ,IAAI;YACf,MAAMrD,KAAK,KAAK,CAACqD,QAAQ,IAAI,CAAC,IAAI,GAAGvD;YACrC,KAAKE,KAAK,KAAK,CAACqD,QAAQ,IAAI,CAAC,GAAG,GAAGvD;YACnC,OAAOE,KAAK,KAAK,CAACqD,QAAQ,IAAI,CAAC,KAAK,GAAGvD;YACvC,QAAQE,KAAK,KAAK,CAACqD,QAAQ,IAAI,CAAC,MAAM,GAAGvD;QAC3C;IACF;AACF;AAEO,MAAMsE,uCAAuC,CAClDjB,MACArD;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOqD;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMnD,KAAK,KAAK,CAACmD,KAAK,IAAI,GAAGrD;QAC7B,KAAKE,KAAK,KAAK,CAACmD,KAAK,GAAG,GAAGrD;QAC3B,OAAOE,KAAK,KAAK,CAACmD,KAAK,KAAK,GAAGrD;QAC/B,QAAQE,KAAK,KAAK,CAACmD,KAAK,MAAM,GAAGrD;IACnC;AACF"}
+{"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\nimport { pixelBboxToRect } from '@/ai-model/workflows/grounding/locate-result-rect';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PixelBbox,\n  PlanningLocateParam,\n  PlanningLocateParamWithLocatedPixelBbox,\n  Rect,\n  ScrollParam,\n  Size,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport {\n  convertPngBase64ToJpeg,\n  createImgBase64ByFormat,\n  imageInfoOfBase64,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, ifInBrowser, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst agentDebug = getDebug('agent');\nconst screenshotDataUrlPattern = /^data:image\\/[a-zA-Z0-9.+-]+;base64,/i;\n\nconst inferBase64ImageFormat = (base64Body: string) => {\n  if (base64Body.startsWith('iVBORw0KGgo')) {\n    return 'png';\n  }\n  return 'jpeg';\n};\n\nconst normalizeScreenshotBase64 = (screenshotBase64: string) => {\n  const trimmedBase64 = screenshotBase64.trim();\n  if (screenshotDataUrlPattern.test(trimmedBase64)) {\n    return trimmedBase64;\n  }\n\n  const base64Body = trimmedBase64.replace(/\\s/g, '');\n  assert(base64Body, 'screenshotBase64 must include image data');\n  return createImgBase64ByFormat(\n    inferBase64ImageFormat(base64Body),\n    base64Body,\n  );\n};\n\nconst legacyScrollTypeMap = {\n  once: 'singleAction',\n  untilBottom: 'scrollToBottom',\n  untilTop: 'scrollToTop',\n  untilRight: 'scrollToRight',\n  untilLeft: 'scrollToLeft',\n} as const;\n\nexport const normalizeScrollType = (\n  scrollType: string | undefined,\n): ScrollParam['scrollType'] | undefined => {\n  if (!scrollType) {\n    return undefined;\n  }\n\n  if (scrollType in legacyScrollTypeMap) {\n    return legacyScrollTypeMap[scrollType as keyof typeof legacyScrollTypeMap];\n  }\n\n  return scrollType as ScrollParam['scrollType'];\n};\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  } else {\n    // For screenshots that do not need shrinking, convert PNG to JPEG to reduce the image payload in model requests and reports. (Shrunk images are already JPEG.)\n    // This mainly covers Android's default screenshot path, which produces PNG screenshots.\n    // Compared with conversion on Android, centralizing it here means each platform does not need to handle screenshot formats itself, and allows future output formats such as WebP.\n    // Built-in paths that already output JPEG are unaffected, and custom devices that output JPEG will not be compressed again.\n    // The Web platform already outputs JPEG, so it does not enter this branch. Other built-in device platforms run in Node, where Sharp conversion is fast enough that its extra cost is negligible.\n    const outputScreenshotBase64 = await convertPngBase64ToJpeg(\n      screenshotBase64,\n      90,\n    );\n\n    return {\n      shotSize: {\n        width: imgWidth,\n        height: imgHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(\n        outputScreenshotBase64,\n        screenshotCapturedAt,\n      ),\n      shrunkShotToLogicalRatio,\n    };\n  }\n}\n\nexport async function createScreenshotBoundUIContext(\n  screenshotBase64: string,\n  opt: {\n    screenshotSize?: Size;\n  },\n): Promise<UIContext> {\n  const normalizedScreenshotBase64 =\n    normalizeScreenshotBase64(screenshotBase64);\n  const actualScreenshotSize = await imageInfoOfBase64(\n    normalizedScreenshotBase64,\n  );\n  if (\n    opt.screenshotSize &&\n    (opt.screenshotSize.width !== actualScreenshotSize.width ||\n      opt.screenshotSize.height !== actualScreenshotSize.height)\n  ) {\n    agentDebug(\n      'describeElementAtPoint screenshotSize mismatch, use actual size',\n      {\n        provided: opt.screenshotSize,\n        actual: actualScreenshotSize,\n      },\n    );\n  }\n\n  return {\n    screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n    shotSize: actualScreenshotSize,\n    shrunkShotToLogicalRatio: 1,\n    _isFrozen: true,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\ntype NormalizeFilePathsOptions = {\n  fileExists?: (path: string) => boolean;\n  isInBrowser?: boolean;\n  resolvePath?: (path: string) => string;\n  wslDistroName?: string;\n  cwd?: string;\n};\n\nexport function normalizeFilePaths(\n  files: string[],\n  options: NormalizeFilePathsOptions = {},\n): string[] {\n  const {\n    fileExists = existsSync,\n    isInBrowser = ifInBrowser,\n    resolvePath = resolve,\n    wslDistroName = process.env.WSL_DISTRO_NAME,\n    cwd = process.cwd(),\n  } = options;\n\n  if (isInBrowser) {\n    throw new Error('File chooser is not supported in browser environment');\n  }\n\n  return files.map((file) => {\n    const absolutePath = resolvePath(file);\n    if (!fileExists(absolutePath)) {\n      throw new Error(\n        `File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`,\n      );\n    }\n\n    if (!wslDistroName) {\n      return absolutePath;\n    }\n\n    const wslMount = absolutePath.match(/^\\/mnt\\/([a-z])\\//);\n    if (wslMount) {\n      return `${wslMount[1].toUpperCase()}:\\\\${absolutePath.slice(7).replace(/\\//g, '\\\\')}`;\n    }\n\n    return `\\\\\\\\wsl$\\\\${wslDistroName}${absolutePath.replace(/\\//g, '\\\\')}`;\n  });\n}\n\nexport function isPixelBbox(value: unknown): value is PixelBbox {\n  return (\n    Array.isArray(value) &&\n    value.length === 4 &&\n    value.every((item) => typeof item === 'number' && Number.isFinite(item))\n  );\n}\n\ntype PlanningLocateParamWithMaybeLocatedPixelBbox = PlanningLocateParam & {\n  locatedPixelBbox?: unknown;\n};\n\nexport function ifPlanLocateParamHasLocatedPixelBbox(\n  planLocateParam: PlanningLocateParamWithMaybeLocatedPixelBbox,\n): planLocateParam is PlanningLocateParamWithLocatedPixelBbox {\n  return isPixelBbox(planLocateParam.locatedPixelBbox);\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParamWithLocatedPixelBbox,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  const rect = pixelBboxToRect(planLocateParam.locatedPixelBbox);\n\n  const element = generateElementByRect(\n    rect,\n    typeof planLocateParam.prompt === 'string'\n      ? planLocateParam.prompt\n      : planLocateParam.prompt?.prompt || '',\n  );\n  return element;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["agentDebug","getDebug","screenshotDataUrlPattern","inferBase64ImageFormat","base64Body","normalizeScreenshotBase64","screenshotBase64","trimmedBase64","assert","createImgBase64ByFormat","legacyScrollTypeMap","normalizeScrollType","scrollType","commonContextParser","interfaceInstance","_opt","debug","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","outputScreenshotBase64","convertPngBase64ToJpeg","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","actualScreenshotSize","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","normalizeFilePaths","files","options","fileExists","existsSync","isInBrowser","ifInBrowser","resolvePath","resolve","wslDistroName","process","cwd","file","absolutePath","wslMount","isPixelBbox","value","Array","item","ifPlanLocateParamHasLocatedPixelBbox","planLocateParam","matchElementFromPlan","rect","pixelBboxToRect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;;;;;;;;;;AAqCA,MAAMA,aAAaC,SAAS;AAC5B,MAAMC,2BAA2B;AAEjC,MAAMC,yBAAyB,CAACC;IAC9B,IAAIA,WAAW,UAAU,CAAC,gBACxB,OAAO;IAET,OAAO;AACT;AAEA,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,gBAAgBD,iBAAiB,IAAI;IAC3C,IAAIJ,yBAAyB,IAAI,CAACK,gBAChC,OAAOA;IAGT,MAAMH,aAAaG,cAAc,OAAO,CAAC,OAAO;IAChDC,OAAOJ,YAAY;IACnB,OAAOK,wBACLN,uBAAuBC,aACvBA;AAEJ;AAEA,MAAMM,sBAAsB;IAC1B,MAAM;IACN,aAAa;IACb,UAAU;IACV,YAAY;IACZ,WAAW;AACb;AAEO,MAAMC,sBAAsB,CACjCC;IAEA,IAAI,CAACA,YACH;IAGF,IAAIA,cAAcF,qBAChB,OAAOA,mBAAmB,CAACE,WAA+C;IAG5E,OAAOA;AACT;AAEO,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQf,SAAS;IAEvBO,OAAOM,mBAAmB;IAE1BE,MAAM;IACN,MAAMC,cAAcH,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNE,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMG,gBAAgB,MAAML,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOM,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIL,MAAM,CAAC,MAAM,EAAEI,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMf,mBAAmB,MAAMQ,kBAAkB,gBAAgB;IACjE,MAAMW,uBAAuBC,KAAK,GAAG;IACrClB,OAAOF,kBAAmB;IAG1BU,MAAM;IACN,MAAM,EAAE,OAAOW,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,kBAAkBvB;IAE1B,IAAI,CAACiB,OAAO,QAAQ,CAACI,aAAa,CAACJ,OAAO,QAAQ,CAACK,YACjD,MAAM,IAAIJ,MACR,CAAC,wFAAwF,EAAEG,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIJ,MACR,CAAC,0FAA0F,EAAEG,SAAS,UAAU,EAAEC,WAAW;IAGjIZ,MAAM,yBAAyBW,UAAU,KAAKC;IAM9C,MAAME,oBAAoBV,eAAeC;IACzC,MAAMU,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBZ;IAExB,IAAIU,sBAAsBC,sBAAsB;QAC9Cf,MACE,CAAC,4CAA4C,EAAEI,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAES,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBX;IAEtB;IAEA,MAAMY,mBAAmBlB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACQ,OAAO,QAAQ,CAACU,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIT,MACR,CAAC,wEAAwE,EAAES,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBhB,MAAM,mBAAmBkB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCjB,MAAM,4BAA4BmB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CjB,MACE,CAAC,mCAAmC,EAAEiB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,gBAAgBlC,kBAAkB;YAC5D,OAAO8B;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,eAAe,MAAM,CAACF,eAAed;YACjDU;QACF;IACF;IAAO;QAML,MAAMO,yBAAyB,MAAMC,uBACnCrC,kBACA;QAGF,OAAO;YACL,UAAU;gBACR,OAAOqB;gBACP,QAAQC;YACV;YACA,eAAeM;YACf,YAAYO,eAAe,MAAM,CAC/BC,wBACAjB;YAEFU;QACF;IACF;AACF;AAEO,eAAeS,+BACpBtC,gBAAwB,EACxBuC,GAEC;IAED,MAAMC,6BACJzC,0BAA0BC;IAC5B,MAAMyC,uBAAuB,MAAMlB,kBACjCiB;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKE,qBAAqB,KAAK,IACtDF,IAAI,cAAc,CAAC,MAAM,KAAKE,qBAAqB,MAAK,GAE1D/C,WACE,mEACA;QACE,UAAU6C,IAAI,cAAc;QAC5B,QAAQE;IACV;IAIJ,OAAO;QACL,YAAYN,eAAe,MAAM,CAACK,4BAA4BpB,KAAK,GAAG;QACtE,UAAUqB;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASC,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAUO,SAASG,mBACdC,KAAe,EACfC,UAAqC,CAAC,CAAC;IAEvC,MAAM,EACJC,aAAaC,UAAU,EACvBC,cAAcC,WAAW,EACzBC,cAAcC,OAAO,EACrBC,gBAAgBC,QAAQ,GAAG,CAAC,eAAe,EAC3CC,MAAMD,QAAQ,GAAG,EAAE,EACpB,GAAGR;IAEJ,IAAIG,aACF,MAAM,IAAI1C,MAAM;IAGlB,OAAOsC,MAAM,GAAG,CAAC,CAACW;QAChB,MAAMC,eAAeN,YAAYK;QACjC,IAAI,CAACT,WAAWU,eACd,MAAM,IAAIlD,MACR,CAAC,gBAAgB,EAAEiD,KAAK,eAAe,EAAEC,aAAa,6BAA6B,EAAEF,KAAK;QAI9F,IAAI,CAACF,eACH,OAAOI;QAGT,MAAMC,WAAWD,aAAa,KAAK,CAAC;QACpC,IAAIC,UACF,OAAO,GAAGA,QAAQ,CAAC,EAAE,CAAC,WAAW,GAAG,GAAG,EAAED,aAAa,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,OAAO;QAGvF,OAAO,CAAC,UAAU,EAAEJ,gBAAgBI,aAAa,OAAO,CAAC,OAAO,OAAO;IACzE;AACF;AAEO,SAASE,YAAYC,KAAc;IACxC,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,OAAS,AAAgB,YAAhB,OAAOA,QAAqBxD,OAAO,QAAQ,CAACwD;AAEtE;AAMO,SAASC,qCACdC,eAA6D;IAE7D,OAAOL,YAAYK,gBAAgB,gBAAgB;AACrD;AAEO,SAASC,qBACdD,eAAwD;IAExD,IAAI,CAACA,iBACH;IAGF,MAAME,OAAOC,gBAAgBH,gBAAgB,gBAAgB;IAE7D,MAAMI,UAAUC,sBACdH,MACA,AAAkC,YAAlC,OAAOF,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;IAExC,OAAOI;AACT;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMT,OACJ,MAAMK,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNhD,KAAK,KAAK,CAAC8C,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpC9C,KAAK,KAAK,CAAC8C,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOO,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,8BAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAlD;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOkD;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNhD,KAAK,KAAK,CAACgD,QAAQ,MAAM,CAAC,EAAE,GAAGlD;YAC/BE,KAAK,KAAK,CAACgD,QAAQ,MAAM,CAAC,EAAE,GAAGlD;SAChC;QACD,MAAM;YACJ,GAAGkD,QAAQ,IAAI;YACf,MAAMhD,KAAK,KAAK,CAACgD,QAAQ,IAAI,CAAC,IAAI,GAAGlD;YACrC,KAAKE,KAAK,KAAK,CAACgD,QAAQ,IAAI,CAAC,GAAG,GAAGlD;YACnC,OAAOE,KAAK,KAAK,CAACgD,QAAQ,IAAI,CAAC,KAAK,GAAGlD;YACvC,QAAQE,KAAK,KAAK,CAACgD,QAAQ,IAAI,CAAC,MAAM,GAAGlD;QAC3C;IACF;AACF;AAEO,MAAMiE,uCAAuC,CAClDjB,MACAhD;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOgD;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAM9C,KAAK,KAAK,CAAC8C,KAAK,IAAI,GAAGhD;QAC7B,KAAKE,KAAK,KAAK,CAAC8C,KAAK,GAAG,GAAGhD;QAC3B,OAAOE,KAAK,KAAK,CAAC8C,KAAK,KAAK,GAAGhD;QAC/B,QAAQE,KAAK,KAAK,CAAC8C,KAAK,MAAM,GAAGhD;IACnC;AACF"}