@aiscene/core 9.0.6 → 9.0.8

package/dist/lib/agent/ui-observer.js

@@ -0,0 +1,425 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    UIObservationImpl: ()=>UIObservationImpl,
+    uiContextFromObservationRecord: ()=>uiContextFromObservationRecord,
+    UIObserverImpl: ()=>UIObserverImpl
+});
+const observation_record_namespaceObject = require("@aiscene/shared/agent-tools/observation-record");
+const logger_namespaceObject = require("@aiscene/shared/logger");
+const utils_namespaceObject = require("@aiscene/shared/utils");
+const external_screenshot_item_js_namespaceObject = require("../screenshot-item.js");
+const external_screenshot_preparation_js_namespaceObject = require("./screenshot-preparation.js");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = (0, logger_namespaceObject.getDebug)('ui-observer');
+const warnObserver = (0, logger_namespaceObject.getDebug)('ui-observer', {
+    console: true
+});
+const DEFAULT_INTERVAL_MS = 1000;
+const MIN_INTERVAL_MS = 200;
+const DEFAULT_MAX_FRAMES = 30;
+const FIRST_FRAME_TIMEOUT_MS = 3000;
+const DEFAULT_WATCHDOG_MS = 300000;
+const MAX_FRAMES_PER_RECORD = 50;
+const DECODE_BATCH_SIZE = 4;
+class UIObservationImpl {
+    get frameCount() {
+        return this.record.frames.length;
+    }
+    get startedAt() {
+        return this.record.startedAt;
+    }
+    get endedAt() {
+        return this.record.endedAt;
+    }
+    ensureUsable() {
+        (0, utils_namespaceObject.assert)(!this.disposed, 'UI observation has been disposed');
+    }
+    ensureFixedWindowOptions(options) {
+        (0, utils_namespaceObject.assert)(options?.domIncluded === void 0, 'UIObservation does not support domIncluded because it only evaluates recorded screenshots');
+    }
+    async aiQuery(demand, options) {
+        this.ensureUsable();
+        this.ensureFixedWindowOptions(options);
+        return this.insight.aiQuery(demand, options);
+    }
+    async aiBoolean(prompt, options) {
+        this.ensureUsable();
+        this.ensureFixedWindowOptions(options);
+        return this.insight.aiBoolean(prompt, options);
+    }
+    async aiNumber(prompt, options) {
+        this.ensureUsable();
+        this.ensureFixedWindowOptions(options);
+        return this.insight.aiNumber(prompt, options);
+    }
+    async aiString(prompt, options) {
+        this.ensureUsable();
+        this.ensureFixedWindowOptions(options);
+        return this.insight.aiString(prompt, options);
+    }
+    async aiAsk(prompt, options) {
+        this.ensureUsable();
+        this.ensureFixedWindowOptions(options);
+        return this.insight.aiAsk(prompt, options);
+    }
+    async aiAssert(assertion, message, options) {
+        this.ensureUsable();
+        this.ensureFixedWindowOptions(options);
+        return this.insight.aiAssert(assertion, message, options);
+    }
+    async exportRecord() {
+        this.ensureUsable();
+        return (0, observation_record_namespaceObject.cloneUIObservationRecord)(this.record);
+    }
+    async dispose() {
+        if (this.disposed) return;
+        this.disposeRecord?.();
+        this.disposed = true;
+        this.onDisposed?.();
+    }
+    constructor(record, insight, disposeRecord, onDisposed){
+        _define_property(this, "insight", void 0);
+        _define_property(this, "disposeRecord", void 0);
+        _define_property(this, "onDisposed", void 0);
+        _define_property(this, "disposed", void 0);
+        _define_property(this, "record", void 0);
+        this.insight = insight;
+        this.disposeRecord = disposeRecord;
+        this.onDisposed = onDisposed;
+        this.disposed = false;
+        this.record = (0, observation_record_namespaceObject.cloneUIObservationRecord)(record);
+    }
+}
+function isImageDataUrl(value) {
+    return 'string' == typeof value && /^data:image\/(?:png|jpe?g);base64,/i.test(value);
+}
+class UIObserverImpl {
+    get bufferedFrameCount() {
+        return this.frames.length;
+    }
+    async start() {
+        (0, utils_namespaceObject.assert)(!this.loopPromise && !this.stopped, 'observer has already started');
+        this.startedAt = Date.now();
+        try {
+            this.source = await this.deps.openFrameSource() ?? null;
+        } catch (error) {
+            debug(`frame source unavailable, using screenshot fallback: ${error}`);
+            this.source = null;
+        }
+        this.usingFallback = !this.source;
+        if (this.usingFallback) debug('no continuous frame source; sampling via plain screenshots');
+        else {
+            const waitStart = Date.now();
+            while(!this.source.latest() && Date.now() - waitStart < FIRST_FRAME_TIMEOUT_MS)await new Promise((resolve)=>setTimeout(resolve, 50));
+            if (!this.source.latest()) debug(`no first frame within ${FIRST_FRAME_TIMEOUT_MS}ms; starting anyway`);
+        }
+        await this.captureOnce();
+        this.loopPromise = this.runLoop();
+        if (this.watchdogMs > 0) {
+            this.watchdogTimer = setTimeout(()=>{
+                warnObserver(`UIObserver auto-stopped after ${this.watchdogMs}ms. Call observer.stop() explicitly to avoid this.`);
+                this.stop().catch(()=>{});
+            }, this.watchdogMs);
+            if ('function' == typeof this.watchdogTimer.unref) this.watchdogTimer.unref();
+        }
+    }
+    stop() {
+        if (!this.stopPromise) this.stopPromise = this.finalizeStop();
+        return this.stopPromise;
+    }
+    async finalizeStop() {
+        this.stopped = true;
+        if (this.watchdogTimer) {
+            clearTimeout(this.watchdogTimer);
+            this.watchdogTimer = null;
+        }
+        await this.loopPromise;
+        try {
+            if (this.frames.length > 0) this.persistPromise = this.persistUnstoredFrames().catch((error)=>{
+                debug(`frame persistence failed, will retry during export: ${error}`);
+            });
+            const representativePromise = this.deps.capturePreparedRepresentative();
+            const [, representative] = await Promise.all([
+                this.persistPromise,
+                representativePromise
+            ]);
+            const lastFrame = this.frames.at(-1);
+            if (this.source && lastFrame?.persisted) {
+                this.representativeFrame = {
+                    ...lastFrame.persisted,
+                    capturedAt: lastFrame.capturedAt
+                };
+                representative.screenshot = external_screenshot_item_js_namespaceObject.ScreenshotItem.fromFile(this.writer.resolveFramePath(lastFrame.persisted), lastFrame.persisted.mimeType, lastFrame.capturedAt);
+                debug('representative screenshot aligned with last sampled frame');
+            }
+            this.representative = representative;
+            const endedAt = Date.now();
+            const record = await this.finalizeRecord(endedAt);
+            this.observation = new UIObservationImpl(record, this.deps.createInsight(record), ()=>this.writer.dispose(), this.deps.onDisposed);
+            return this.observation;
+        } finally{
+            if (this.source) try {
+                await this.source.stop();
+            } catch (error) {
+                debug(`error stopping frame source: ${error}`);
+            }
+            debug(`observation stopped with ${this.frames.length} buffered frames (+1 representative)`);
+            this.deps.onStopped?.();
+        }
+    }
+    async finalizeRecord(endedAt) {
+        (0, utils_namespaceObject.assert)(this.stopped && this.representative, 'observation must be stopped before finalizing the observed window');
+        if (this.persistPromise) {
+            await this.persistPromise;
+            this.persistPromise = null;
+        }
+        await this.persistUnstoredFrames();
+        const sampledFrames = this.frames.map((frame)=>{
+            (0, utils_namespaceObject.assert)(frame.persisted, 'observation frame was not persisted');
+            return {
+                ...frame.persisted,
+                capturedAt: frame.capturedAt
+            };
+        });
+        if (!this.representativeFrame) {
+            const representative = this.representative;
+            this.representativeFrame = this.writer.persistFrame(representative.screenshot.base64, representative.screenshot.capturedAt);
+            representative.screenshot = external_screenshot_item_js_namespaceObject.ScreenshotItem.fromFile(this.writer.resolveFramePath(this.representativeFrame), this.representativeFrame.mimeType, this.representativeFrame.capturedAt);
+        }
+        const frames = [
+            ...sampledFrames,
+            this.representativeFrame
+        ];
+        if (frames.length > MAX_FRAMES_PER_RECORD) warnObserver(`WARNING: exporting ${frames.length} frames (soft limit ${MAX_FRAMES_PER_RECORD}). Running insight against this observation sends every frame to the model; consider increasing intervalMs or decreasing maxFrames to reduce token cost.`);
+        debug(`exporting ${frames.length} file-backed observation frames (${this.persistedByRef.size} decoded source refs)`);
+        const metadata = {
+            startedAt: this.startedAt,
+            endedAt,
+            shotSize: {
+                ...this.representative.shotSize
+            },
+            shrunkShotToLogicalRatio: this.representative.shrunkShotToLogicalRatio
+        };
+        return this.writer.finalize(frames, metadata);
+    }
+    async dispose() {
+        if (this.disposed) return;
+        try {
+            if (this.stopPromise) await this.stopPromise;
+            else if (!this.stopped) await this.stop();
+        } finally{
+            if (this.observation) await this.observation.dispose();
+            else {
+                this.writer.dispose();
+                this.deps.onDisposed?.();
+            }
+            this.frames = [];
+            this.persistedByRef.clear();
+            this.observation = null;
+            this.disposed = true;
+        }
+    }
+    async persistUnstoredFrames() {
+        const uniqueFrames = this.dedupeRefs(this.frames.filter((frame)=>!frame.persisted));
+        const uncachedFrames = uniqueFrames.filter((frame)=>!this.persistedByRef.has(frame.ref));
+        for(let start = 0; start < uncachedFrames.length; start += DECODE_BATCH_SIZE){
+            const batch = uncachedFrames.slice(start, start + DECODE_BATCH_SIZE);
+            const decoded = this.source ? await this.source.decode(batch) : batch.map((frame)=>{
+                (0, utils_namespaceObject.assert)(isImageDataUrl(frame.ref), 'fallback observation frame must be an image data URL');
+                return frame.ref;
+            });
+            (0, utils_namespaceObject.assert)(decoded.length === batch.length, 'frame source decode() must return one image per frame handle');
+            const preparedDataUrls = await Promise.all(decoded.map((dataUrl)=>(0, external_screenshot_preparation_js_namespaceObject.prepareScreenshotForPersistence)(dataUrl, {
+                    shrinkFactor: this.screenshotShrinkFactor
+                })));
+            for(let index = 0; index < batch.length; index++)this.persistedByRef.set(batch[index].ref, this.writer.persistFrame(preparedDataUrls[index], batch[index].capturedAt));
+        }
+        for (const frame of this.frames)frame.persisted ??= this.persistedByRef.get(frame.ref);
+        if (uncachedFrames.length > 0) debug(`decoded and persisted ${uncachedFrames.length} source frames`);
+    }
+    async captureOnce() {
+        try {
+            if (this.source) {
+                const frame = this.source.latest();
+                if (!frame) return;
+                if (isImageDataUrl(frame.ref)) {
+                    const preparedDataUrl = await (0, external_screenshot_preparation_js_namespaceObject.prepareScreenshotForPersistence)(frame.ref, {
+                        shrinkFactor: this.screenshotShrinkFactor
+                    });
+                    const persisted = this.writer.persistFrame(preparedDataUrl, frame.capturedAt);
+                    this.pushFrame({
+                        ref: persisted.path,
+                        capturedAt: frame.capturedAt,
+                        persisted
+                    });
+                } else this.pushFrame(frame);
+                return;
+            }
+            const preparedDataUrl = await (0, external_screenshot_preparation_js_namespaceObject.prepareScreenshotForPersistence)(await this.deps.captureRawScreenshot(), {
+                shrinkFactor: this.screenshotShrinkFactor
+            });
+            const persisted = this.writer.persistFrame(preparedDataUrl, Date.now());
+            this.pushFrame({
+                ref: persisted.path,
+                capturedAt: persisted.capturedAt,
+                persisted
+            });
+        } catch (error) {
+            debug(`frame capture failed, skipping tick: ${error}`);
+        }
+    }
+    async runLoop() {
+        while(!this.stopped){
+            const tickStart = Date.now();
+            await this.captureOnce();
+            while(!this.stopped && Date.now() - tickStart < this.intervalMs)await new Promise((resolve)=>setTimeout(resolve, 50));
+        }
+    }
+    pushFrame(frame) {
+        this.frames.push(frame);
+        if (this.frames.length > this.maxFrames) {
+            this.frames = this.thinBuffer(this.frames);
+            this.writer.pruneFrames(this.frames.flatMap((retainedFrame)=>retainedFrame.persisted ? [
+                    retainedFrame.persisted
+                ] : []));
+            debug(`frame buffer thinned to ${this.frames.length} frames`);
+        }
+    }
+    thinBuffer(frames) {
+        if (frames.length <= 1) return frames;
+        const isChangePoint = new Array(frames.length).fill(false);
+        isChangePoint[0] = true;
+        for(let index = 1; index < frames.length; index++)if (frames[index].ref !== frames[index - 1].ref) isChangePoint[index] = true;
+        isChangePoint[frames.length - 1] = true;
+        let result = [];
+        let staticCounter = 0;
+        for(let index = 0; index < frames.length; index++)if (isChangePoint[index]) {
+            result.push(frames[index]);
+            staticCounter = 0;
+        } else if (staticCounter % 2 === 0) {
+            result.push(frames[index]);
+            staticCounter++;
+        } else staticCounter++;
+        if (result.length > this.maxFrames) {
+            const step = result.length / this.maxFrames;
+            const sampled = [];
+            for(let index = 0; index < this.maxFrames; index++)sampled.push(result[Math.floor(index * step)]);
+            sampled[this.maxFrames - 1] = result[result.length - 1];
+            result = sampled;
+        }
+        return result;
+    }
+    dedupeRefs(frames) {
+        const seen = new Set();
+        const result = [];
+        for (const frame of frames)if (!seen.has(frame.ref)) {
+            seen.add(frame.ref);
+            result.push(frame);
+        }
+        return result;
+    }
+    constructor(deps, opt){
+        _define_property(this, "deps", void 0);
+        _define_property(this, "frames", void 0);
+        _define_property(this, "source", void 0);
+        _define_property(this, "usingFallback", void 0);
+        _define_property(this, "stopped", void 0);
+        _define_property(this, "disposed", void 0);
+        _define_property(this, "loopPromise", void 0);
+        _define_property(this, "stopPromise", void 0);
+        _define_property(this, "representative", void 0);
+        _define_property(this, "representativeFrame", void 0);
+        _define_property(this, "watchdogTimer", void 0);
+        _define_property(this, "persistPromise", void 0);
+        _define_property(this, "persistedByRef", void 0);
+        _define_property(this, "observation", void 0);
+        _define_property(this, "startedAt", void 0);
+        _define_property(this, "intervalMs", void 0);
+        _define_property(this, "maxFrames", void 0);
+        _define_property(this, "watchdogMs", void 0);
+        _define_property(this, "screenshotShrinkFactor", void 0);
+        _define_property(this, "writer", void 0);
+        this.deps = deps;
+        this.frames = [];
+        this.source = null;
+        this.usingFallback = false;
+        this.stopped = false;
+        this.disposed = false;
+        this.loopPromise = null;
+        this.stopPromise = null;
+        this.representative = null;
+        this.representativeFrame = null;
+        this.watchdogTimer = null;
+        this.persistPromise = null;
+        this.persistedByRef = new Map();
+        this.observation = null;
+        this.startedAt = 0;
+        this.intervalMs = Math.max(MIN_INTERVAL_MS, opt?.intervalMs ?? DEFAULT_INTERVAL_MS);
+        this.maxFrames = Math.max(2, opt?.maxFrames ?? DEFAULT_MAX_FRAMES);
+        this.watchdogMs = opt?.watchdogMs ?? DEFAULT_WATCHDOG_MS;
+        this.screenshotShrinkFactor = deps.screenshotShrinkFactor ?? 1;
+        this.writer = deps.observationRecordWriter ?? new observation_record_namespaceObject.UIObservationRecordWriter();
+    }
+}
+function uiContextFromObservationRecord(record) {
+    (0, utils_namespaceObject.assert)('midscene_ui_observation' === record.type && 1 === record.version, 'invalid UI observation record type or version');
+    (0, utils_namespaceObject.assert)(record.frames.length > 0, 'UI observation record contains no frames');
+    (0, utils_namespaceObject.assert)(Number.isFinite(record.shotSize.width) && record.shotSize.width > 0, 'UI observation record shot width must be positive');
+    (0, utils_namespaceObject.assert)(Number.isFinite(record.shotSize.height) && record.shotSize.height > 0, 'UI observation record shot height must be positive');
+    (0, utils_namespaceObject.assert)(Number.isFinite(record.shrunkShotToLogicalRatio) && record.shrunkShotToLogicalRatio > 0, 'UI observation record screenshot ratio must be positive');
+    const screenshotSequence = record.frames.map((frame)=>external_screenshot_item_js_namespaceObject.ScreenshotItem.fromFile(frame.path, frame.mimeType, frame.capturedAt));
+    return {
+        screenshot: screenshotSequence[screenshotSequence.length - 1],
+        screenshotSequence,
+        shotSize: {
+            ...record.shotSize
+        },
+        shrunkShotToLogicalRatio: record.shrunkShotToLogicalRatio
+    };
+}
+exports.UIObservationImpl = __webpack_exports__.UIObservationImpl;
+exports.UIObserverImpl = __webpack_exports__.UIObserverImpl;
+exports.uiContextFromObservationRecord = __webpack_exports__.uiContextFromObservationRecord;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "UIObservationImpl",
+    "UIObserverImpl",
+    "uiContextFromObservationRecord"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=ui-observer.js.map

package/dist/lib/agent/ui-observer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/ui-observer.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/ui-observer.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  type UIObservationRecordMetadata,\n  UIObservationRecordWriter,\n  cloneUIObservationRecord,\n} from '@aiscene/shared/agent-tools/observation-record';\nimport type {\n  BaseUIObserverOptions,\n  UIObservationFrame,\n  UIObservationRecord,\n} from '@aiscene/shared/agent-tools/types';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert } from '@aiscene/shared/utils';\nimport type { TUserPrompt } from '../common';\nimport type { DeviceFrameRef, DeviceFrameSource } from '../device';\nimport { ScreenshotItem } from '../screenshot-item';\nimport type {\n  AgentAssertResult,\n  InsightAPI,\n  ObservationAssertOptions,\n  ObservationQueryOptions,\n  ServiceExtractParam,\n  UIContext,\n} from '../types';\nimport { prepareScreenshotForPersistence } from './screenshot-preparation';\n\nconst debug = getDebug('ui-observer');\nconst warnObserver = getDebug('ui-observer', { console: true });\n\nconst DEFAULT_INTERVAL_MS = 1000;\nconst MIN_INTERVAL_MS = 200;\nconst DEFAULT_MAX_FRAMES = 30;\nconst FIRST_FRAME_TIMEOUT_MS = 3000;\nconst DEFAULT_WATCHDOG_MS = 5 * 60 * 1000;\nconst MAX_FRAMES_PER_RECORD = 50;\nconst DECODE_BATCH_SIZE = 4;\n\n/** Options for a UI observation window. */\nexport type UIObserverOption = BaseUIObserverOptions;\n\ninterface UIObserverDeps {\n  openFrameSource: () => Promise<DeviceFrameSource | undefined>;\n  captureRawScreenshot: () => Promise<string>;\n  capturePreparedRepresentative: () => Promise<UIContext>;\n  createInsight: (record: UIObservationRecord) => InsightAPI;\n  onStopped?: () => void;\n  onDisposed?: () => void;\n  screenshotShrinkFactor?: number;\n  /** Test/internal persistence override; not part of the Agent SDK options. */\n  observationRecordWriter?: UIObservationRecordWriter;\n}\n\n/** A fixed screen-recording window that supports read-only AI insights. */\nexport interface UIObservation\n  extends InsightAPI<ObservationQueryOptions, ObservationAssertOptions> {\n  /** Number of captured frames in the fixed observation window. */\n  readonly frameCount: number;\n  /** Timestamp when screen sampling started. */\n  readonly startedAt: number;\n  /** Timestamp when screen sampling ended. */\n  readonly endedAt: number;\n  /** Release image files owned by this observation. Failed cleanup is retryable. */\n  dispose(): Promise<void>;\n}\n\n/** Recording lifecycle returned by {@link Agent.startObserving}. */\nexport interface UIObserver {\n  /** Number of frames currently buffered while recording. */\n  readonly bufferedFrameCount: number;\n  /** Stop recording and return its fixed observation window. */\n  stop(): Promise<UIObservation>;\n  /** Stop recording if needed and release its image files. */\n  dispose(): Promise<void>;\n}\n\n/** @internal Concrete fixed-window implementation. */\nexport class UIObservationImpl implements UIObservation {\n  private disposed = false;\n  private readonly record: UIObservationRecord;\n\n  constructor(\n    record: UIObservationRecord,\n    private readonly insight: InsightAPI,\n    private readonly disposeRecord?: () => void,\n    private readonly onDisposed?: () => void,\n  ) {\n    this.record = cloneUIObservationRecord(record);\n  }\n\n  get frameCount(): number {\n    return this.record.frames.length;\n  }\n\n  get startedAt(): number {\n    return this.record.startedAt;\n  }\n\n  get endedAt(): number {\n    return this.record.endedAt;\n  }\n\n  private ensureUsable(): void {\n    assert(!this.disposed, 'UI observation has been disposed');\n  }\n\n  private ensureFixedWindowOptions(options?: { domIncluded?: unknown }): void {\n    assert(\n      options?.domIncluded === undefined,\n      'UIObservation does not support domIncluded because it only evaluates recorded screenshots',\n    );\n  }\n\n  async aiQuery<ReturnType = any>(\n    demand: ServiceExtractParam,\n    options?: ObservationQueryOptions,\n  ): Promise<ReturnType> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiQuery<ReturnType>(demand, options);\n  }\n\n  async aiBoolean(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<boolean> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiBoolean(prompt, options);\n  }\n\n  async aiNumber(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<number> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiNumber(prompt, options);\n  }\n\n  async aiString(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<string> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiString(prompt, options);\n  }\n\n  async aiAsk(\n    prompt: TUserPrompt,\n    options?: ObservationQueryOptions,\n  ): Promise<string> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiAsk(prompt, options);\n  }\n\n  async aiAssert(\n    assertion: TUserPrompt,\n    message?: string,\n    options?: ObservationAssertOptions,\n  ): Promise<AgentAssertResult | undefined> {\n    this.ensureUsable();\n    this.ensureFixedWindowOptions(options);\n    return this.insight.aiAssert(assertion, message, options);\n  }\n\n  /** @internal Used only by the CLI observation artifact adapter. */\n  async exportRecord(): Promise<UIObservationRecord> {\n    this.ensureUsable();\n    return cloneUIObservationRecord(this.record);\n  }\n\n  async dispose(): Promise<void> {\n    if (this.disposed) return;\n    this.disposeRecord?.();\n    this.disposed = true;\n    this.onDisposed?.();\n  }\n}\n\ninterface BufferedFrame extends DeviceFrameRef {\n  /** Present once the frame no longer needs to retain an in-memory data URL. */\n  persisted?: UIObservationFrame;\n}\n\nfunction isImageDataUrl(value: unknown): value is string {\n  return (\n    typeof value === 'string' &&\n    /^data:image\\/(?:png|jpe?g);base64,/i.test(value)\n  );\n}\n\n/**\n * Observe an explicit screen window and produce a fixed UIObservation.\n */\nexport class UIObserverImpl implements UIObserver {\n  private frames: BufferedFrame[] = [];\n  private source: DeviceFrameSource | null = null;\n  private usingFallback = false;\n  private stopped = false;\n  private disposed = false;\n  private loopPromise: Promise<void> | null = null;\n  private stopPromise: Promise<UIObservationImpl> | null = null;\n  private representative: UIContext | null = null;\n  private representativeFrame: UIObservationFrame | null = null;\n  private watchdogTimer: ReturnType<typeof setTimeout> | null = null;\n  private persistPromise: Promise<void> | null = null;\n  private persistedByRef = new Map<unknown, UIObservationFrame>();\n  private observation: UIObservationImpl | null = null;\n  private startedAt = 0;\n  private readonly intervalMs: number;\n  private readonly maxFrames: number;\n  private readonly watchdogMs: number;\n  private readonly screenshotShrinkFactor: number;\n  private readonly writer: UIObservationRecordWriter;\n\n  constructor(\n    private readonly deps: UIObserverDeps,\n    opt?: UIObserverOption,\n  ) {\n    this.intervalMs = Math.max(\n      MIN_INTERVAL_MS,\n      opt?.intervalMs ?? DEFAULT_INTERVAL_MS,\n    );\n    this.maxFrames = Math.max(2, opt?.maxFrames ?? DEFAULT_MAX_FRAMES);\n    this.watchdogMs = opt?.watchdogMs ?? DEFAULT_WATCHDOG_MS;\n    this.screenshotShrinkFactor = deps.screenshotShrinkFactor ?? 1;\n    this.writer =\n      deps.observationRecordWriter ?? new UIObservationRecordWriter();\n  }\n\n  get bufferedFrameCount(): number {\n    return this.frames.length;\n  }\n\n  async start(): Promise<void> {\n    assert(!this.loopPromise && !this.stopped, 'observer has already started');\n    this.startedAt = Date.now();\n    try {\n      this.source = (await this.deps.openFrameSource()) ?? null;\n    } catch (error) {\n      debug(`frame source unavailable, using screenshot fallback: ${error}`);\n      this.source = null;\n    }\n    this.usingFallback = !this.source;\n    if (this.usingFallback) {\n      debug('no continuous frame source; sampling via plain screenshots');\n    } else {\n      const waitStart = Date.now();\n      while (\n        !this.source!.latest() &&\n        Date.now() - waitStart < FIRST_FRAME_TIMEOUT_MS\n      ) {\n        await new Promise((resolve) => setTimeout(resolve, 50));\n      }\n      if (!this.source!.latest()) {\n        debug(\n          `no first frame within ${FIRST_FRAME_TIMEOUT_MS}ms; starting anyway`,\n        );\n      }\n    }\n    await this.captureOnce();\n    this.loopPromise = this.runLoop();\n\n    if (this.watchdogMs > 0) {\n      this.watchdogTimer = setTimeout(() => {\n        warnObserver(\n          `UIObserver auto-stopped after ${this.watchdogMs}ms. Call observer.stop() explicitly to avoid this.`,\n        );\n        this.stop().catch(() => {});\n      }, this.watchdogMs);\n      if (\n        typeof (this.watchdogTimer as { unref?: () => void }).unref ===\n        'function'\n      ) {\n        (this.watchdogTimer as { unref: () => void }).unref();\n      }\n    }\n  }\n\n  stop(): Promise<UIObservationImpl> {\n    if (!this.stopPromise) {\n      this.stopPromise = this.finalizeStop();\n    }\n    return this.stopPromise;\n  }\n\n  private async finalizeStop(): Promise<UIObservationImpl> {\n    this.stopped = true;\n    if (this.watchdogTimer) {\n      clearTimeout(this.watchdogTimer);\n      this.watchdogTimer = null;\n    }\n    await this.loopPromise;\n\n    try {\n      if (this.frames.length > 0) {\n        this.persistPromise = this.persistUnstoredFrames().catch((error) => {\n          debug(`frame persistence failed, will retry during export: ${error}`);\n        });\n      }\n      const representativePromise = this.deps.capturePreparedRepresentative();\n      const [, representative] = await Promise.all([\n        this.persistPromise,\n        representativePromise,\n      ]);\n\n      const lastFrame = this.frames.at(-1);\n      if (this.source && lastFrame?.persisted) {\n        this.representativeFrame = {\n          ...lastFrame.persisted,\n          capturedAt: lastFrame.capturedAt,\n        };\n        representative.screenshot = ScreenshotItem.fromFile(\n          this.writer.resolveFramePath(lastFrame.persisted),\n          lastFrame.persisted.mimeType,\n          lastFrame.capturedAt,\n        );\n        debug('representative screenshot aligned with last sampled frame');\n      }\n      this.representative = representative;\n      const endedAt = Date.now();\n      const record = await this.finalizeRecord(endedAt);\n      this.observation = new UIObservationImpl(\n        record,\n        this.deps.createInsight(record),\n        () => this.writer.dispose(),\n        this.deps.onDisposed,\n      );\n      return this.observation;\n    } finally {\n      if (this.source) {\n        try {\n          await this.source.stop();\n        } catch (error) {\n          debug(`error stopping frame source: ${error}`);\n        }\n      }\n      debug(\n        `observation stopped with ${this.frames.length} buffered frames (+1 representative)`,\n      );\n      this.deps.onStopped?.();\n    }\n  }\n\n  private async finalizeRecord(endedAt: number): Promise<UIObservationRecord> {\n    assert(\n      this.stopped && this.representative,\n      'observation must be stopped before finalizing the observed window',\n    );\n    if (this.persistPromise) {\n      await this.persistPromise;\n      this.persistPromise = null;\n    }\n    await this.persistUnstoredFrames();\n\n    const sampledFrames = this.frames.map((frame) => {\n      assert(frame.persisted, 'observation frame was not persisted');\n      return { ...frame.persisted, capturedAt: frame.capturedAt };\n    });\n\n    if (!this.representativeFrame) {\n      const representative = this.representative!;\n      // The representative UIContext has already been prepared by\n      // commonContextParser().\n      this.representativeFrame = this.writer.persistFrame(\n        representative.screenshot.base64,\n        representative.screenshot.capturedAt,\n      );\n      representative.screenshot = ScreenshotItem.fromFile(\n        this.writer.resolveFramePath(this.representativeFrame),\n        this.representativeFrame.mimeType,\n        this.representativeFrame.capturedAt,\n      );\n    }\n\n    const frames = [...sampledFrames, this.representativeFrame];\n    if (frames.length > MAX_FRAMES_PER_RECORD) {\n      warnObserver(\n        `WARNING: exporting ${frames.length} frames (soft limit ${MAX_FRAMES_PER_RECORD}). Running insight against this observation sends every frame to the model; consider increasing intervalMs or decreasing maxFrames to reduce token cost.`,\n      );\n    }\n    debug(\n      `exporting ${frames.length} file-backed observation frames (${this.persistedByRef.size} decoded source refs)`,\n    );\n    const metadata: UIObservationRecordMetadata = {\n      startedAt: this.startedAt,\n      endedAt,\n      shotSize: { ...this.representative!.shotSize },\n      shrunkShotToLogicalRatio: this.representative!.shrunkShotToLogicalRatio,\n    };\n    return this.writer.finalize(frames, metadata);\n  }\n\n  /** Release writer-owned image files after the observation is no longer needed. */\n  async dispose(): Promise<void> {\n    if (this.disposed) return;\n    try {\n      if (this.stopPromise) {\n        await this.stopPromise;\n      } else if (!this.stopped) {\n        await this.stop();\n      }\n    } finally {\n      if (this.observation) {\n        await this.observation.dispose();\n      } else {\n        this.writer.dispose();\n        this.deps.onDisposed?.();\n      }\n      this.frames = [];\n      this.persistedByRef.clear();\n      this.observation = null;\n      this.disposed = true;\n    }\n  }\n\n  private async persistUnstoredFrames(): Promise<void> {\n    const uniqueFrames = this.dedupeRefs(\n      this.frames.filter((frame) => !frame.persisted),\n    );\n    const uncachedFrames = uniqueFrames.filter(\n      (frame) => !this.persistedByRef.has(frame.ref),\n    );\n\n    for (\n      let start = 0;\n      start < uncachedFrames.length;\n      start += DECODE_BATCH_SIZE\n    ) {\n      const batch = uncachedFrames.slice(start, start + DECODE_BATCH_SIZE);\n      const decoded = this.source\n        ? await this.source.decode(batch)\n        : batch.map((frame) => {\n            assert(\n              isImageDataUrl(frame.ref),\n              'fallback observation frame must be an image data URL',\n            );\n            return frame.ref;\n          });\n      assert(\n        decoded.length === batch.length,\n        'frame source decode() must return one image per frame handle',\n      );\n      const preparedDataUrls = await Promise.all(\n        decoded.map((dataUrl) =>\n          prepareScreenshotForPersistence(dataUrl, {\n            shrinkFactor: this.screenshotShrinkFactor,\n          }),\n        ),\n      );\n      for (let index = 0; index < batch.length; index++) {\n        this.persistedByRef.set(\n          batch[index].ref,\n          this.writer.persistFrame(\n            preparedDataUrls[index],\n            batch[index].capturedAt,\n          ),\n        );\n      }\n    }\n\n    for (const frame of this.frames) {\n      frame.persisted ??= this.persistedByRef.get(frame.ref);\n    }\n    if (uncachedFrames.length > 0) {\n      debug(`decoded and persisted ${uncachedFrames.length} source frames`);\n    }\n  }\n\n  private async captureOnce(): Promise<void> {\n    try {\n      if (this.source) {\n        const frame = this.source.latest();\n        if (!frame) return;\n        if (isImageDataUrl(frame.ref)) {\n          const preparedDataUrl = await prepareScreenshotForPersistence(\n            frame.ref,\n            { shrinkFactor: this.screenshotShrinkFactor },\n          );\n          const persisted = this.writer.persistFrame(\n            preparedDataUrl,\n            frame.capturedAt,\n          );\n          this.pushFrame({\n            ref: persisted.path,\n            capturedAt: frame.capturedAt,\n            persisted,\n          });\n        } else {\n          this.pushFrame(frame);\n        }\n        return;\n      }\n      const preparedDataUrl = await prepareScreenshotForPersistence(\n        await this.deps.captureRawScreenshot(),\n        { shrinkFactor: this.screenshotShrinkFactor },\n      );\n      const persisted = this.writer.persistFrame(preparedDataUrl, Date.now());\n      this.pushFrame({\n        ref: persisted.path,\n        capturedAt: persisted.capturedAt,\n        persisted,\n      });\n    } catch (error) {\n      debug(`frame capture failed, skipping tick: ${error}`);\n    }\n  }\n\n  private async runLoop(): Promise<void> {\n    while (!this.stopped) {\n      const tickStart = Date.now();\n      await this.captureOnce();\n      while (!this.stopped && Date.now() - tickStart < this.intervalMs) {\n        await new Promise((resolve) => setTimeout(resolve, 50));\n      }\n    }\n  }\n\n  private pushFrame(frame: DeviceFrameRef | BufferedFrame): void {\n    this.frames.push(frame);\n    if (this.frames.length > this.maxFrames) {\n      this.frames = this.thinBuffer(this.frames);\n      this.writer.pruneFrames(\n        this.frames.flatMap((retainedFrame) =>\n          retainedFrame.persisted ? [retainedFrame.persisted] : [],\n        ),\n      );\n      debug(`frame buffer thinned to ${this.frames.length} frames`);\n    }\n  }\n\n  private thinBuffer(frames: BufferedFrame[]): BufferedFrame[] {\n    if (frames.length <= 1) return frames;\n    const isChangePoint = new Array(frames.length).fill(false);\n    isChangePoint[0] = true;\n    for (let index = 1; index < frames.length; index++) {\n      if (frames[index].ref !== frames[index - 1].ref) {\n        isChangePoint[index] = true;\n      }\n    }\n    isChangePoint[frames.length - 1] = true;\n\n    let result: BufferedFrame[] = [];\n    let staticCounter = 0;\n    for (let index = 0; index < frames.length; index++) {\n      if (isChangePoint[index]) {\n        result.push(frames[index]);\n        staticCounter = 0;\n      } else if (staticCounter % 2 === 0) {\n        result.push(frames[index]);\n        staticCounter++;\n      } else {\n        staticCounter++;\n      }\n    }\n\n    if (result.length > this.maxFrames) {\n      const step = result.length / this.maxFrames;\n      const sampled: BufferedFrame[] = [];\n      for (let index = 0; index < this.maxFrames; index++) {\n        sampled.push(result[Math.floor(index * step)]);\n      }\n      sampled[this.maxFrames - 1] = result[result.length - 1];\n      result = sampled;\n    }\n    return result;\n  }\n\n  private dedupeRefs(frames: BufferedFrame[]): BufferedFrame[] {\n    const seen = new Set<unknown>();\n    const result: BufferedFrame[] = [];\n    for (const frame of frames) {\n      if (!seen.has(frame.ref)) {\n        seen.add(frame.ref);\n        result.push(frame);\n      }\n    }\n    return result;\n  }\n}\n\n/** Rebuild model-facing temporal context from resolved image file paths. */\nexport function uiContextFromObservationRecord(\n  record: UIObservationRecord,\n): UIContext {\n  assert(\n    record.type === 'midscene_ui_observation' && record.version === 1,\n    'invalid UI observation record type or version',\n  );\n  assert(record.frames.length > 0, 'UI observation record contains no frames');\n  assert(\n    Number.isFinite(record.shotSize.width) && record.shotSize.width > 0,\n    'UI observation record shot width must be positive',\n  );\n  assert(\n    Number.isFinite(record.shotSize.height) && record.shotSize.height > 0,\n    'UI observation record shot height must be positive',\n  );\n  assert(\n    Number.isFinite(record.shrunkShotToLogicalRatio) &&\n      record.shrunkShotToLogicalRatio > 0,\n    'UI observation record screenshot ratio must be positive',\n  );\n  const screenshotSequence = record.frames.map((frame) =>\n    ScreenshotItem.fromFile(frame.path, frame.mimeType, frame.capturedAt),\n  );\n  return {\n    screenshot: screenshotSequence[screenshotSequence.length - 1],\n    screenshotSequence,\n    shotSize: { ...record.shotSize },\n    shrunkShotToLogicalRatio: record.shrunkShotToLogicalRatio,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnObserver","DEFAULT_INTERVAL_MS","MIN_INTERVAL_MS","DEFAULT_MAX_FRAMES","FIRST_FRAME_TIMEOUT_MS","DEFAULT_WATCHDOG_MS","MAX_FRAMES_PER_RECORD","DECODE_BATCH_SIZE","UIObservationImpl","assert","options","undefined","demand","prompt","assertion","message","cloneUIObservationRecord","record","insight","disposeRecord","onDisposed","isImageDataUrl","value","UIObserverImpl","Date","error","waitStart","Promise","resolve","setTimeout","clearTimeout","representativePromise","representative","lastFrame","ScreenshotItem","endedAt","sampledFrames","frame","frames","metadata","uniqueFrames","uncachedFrames","start","batch","decoded","preparedDataUrls","dataUrl","prepareScreenshotForPersistence","index","preparedDataUrl","persisted","tickStart","retainedFrame","isChangePoint","Array","result","staticCounter","step","sampled","Math","seen","Set","deps","opt","Map","UIObservationRecordWriter","uiContextFromObservationRecord","Number","screenshotSequence"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;ACmBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,eAAeD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,eAAe;IAAE,SAAS;AAAK;AAE7D,MAAME,sBAAsB;AAC5B,MAAMC,kBAAkB;AACxB,MAAMC,qBAAqB;AAC3B,MAAMC,yBAAyB;AAC/B,MAAMC,sBAAsB;AAC5B,MAAMC,wBAAwB;AAC9B,MAAMC,oBAAoB;AAyCnB,MAAMC;IAaX,IAAI,aAAqB;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM;IAClC;IAEA,IAAI,YAAoB;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS;IAC9B;IAEA,IAAI,UAAkB;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO;IAC5B;IAEQ,eAAqB;QAC3BC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,IAAI,CAAC,QAAQ,EAAE;IACzB;IAEQ,yBAAyBC,OAAmC,EAAQ;QAC1ED,IAAAA,sBAAAA,MAAAA,AAAAA,EACEC,SAAS,gBAAgBC,QACzB;IAEJ;IAEA,MAAM,QACJC,MAA2B,EAC3BF,OAAiC,EACZ;QACrB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAaE,QAAQF;IAClD;IAEA,MAAM,UACJG,MAAmB,EACnBH,OAAiC,EACf;QAClB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAACG,QAAQH;IACxC;IAEA,MAAM,SACJG,MAAmB,EACnBH,OAAiC,EAChB;QACjB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACG,QAAQH;IACvC;IAEA,MAAM,SACJG,MAAmB,EACnBH,OAAiC,EAChB;QACjB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACG,QAAQH;IACvC;IAEA,MAAM,MACJG,MAAmB,EACnBH,OAAiC,EAChB;QACjB,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAACG,QAAQH;IACpC;IAEA,MAAM,SACJI,SAAsB,EACtBC,OAAgB,EAChBL,OAAkC,EACM;QACxC,IAAI,CAAC,YAAY;QACjB,IAAI,CAAC,wBAAwB,CAACA;QAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAACI,WAAWC,SAASL;IACnD;IAGA,MAAM,eAA6C;QACjD,IAAI,CAAC,YAAY;QACjB,OAAOM,AAAAA,IAAAA,mCAAAA,wBAAAA,AAAAA,EAAyB,IAAI,CAAC,MAAM;IAC7C;IAEA,MAAM,UAAyB;QAC7B,IAAI,IAAI,CAAC,QAAQ,EAAE;QACnB,IAAI,CAAC,aAAa;QAClB,IAAI,CAAC,QAAQ,GAAG;QAChB,IAAI,CAAC,UAAU;IACjB;IAlGA,YACEC,MAA2B,EACVC,OAAmB,EACnBC,aAA0B,EAC1BC,UAAuB,CACxC;;;;QARF,uBAAQ,YAAR;QACA,uBAAiB,UAAjB;aAImBF,OAAO,GAAPA;aACAC,aAAa,GAAbA;aACAC,UAAU,GAAVA;aAPX,QAAQ,GAAG;QASjB,IAAI,CAAC,MAAM,GAAGJ,AAAAA,IAAAA,mCAAAA,wBAAAA,AAAAA,EAAyBC;IACzC;AA4FF;AAOA,SAASI,eAAeC,KAAc;IACpC,OACE,AAAiB,YAAjB,OAAOA,SACP,sCAAsC,IAAI,CAACA;AAE/C;AAKO,MAAMC;IAoCX,IAAI,qBAA6B;QAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM;IAC3B;IAEA,MAAM,QAAuB;QAC3Bd,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QAC3C,IAAI,CAAC,SAAS,GAAGe,KAAK,GAAG;QACzB,IAAI;YACF,IAAI,CAAC,MAAM,GAAI,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,MAAO;QACvD,EAAE,OAAOC,OAAO;YACd3B,MAAM,CAAC,qDAAqD,EAAE2B,OAAO;YACrE,IAAI,CAAC,MAAM,GAAG;QAChB;QACA,IAAI,CAAC,aAAa,GAAG,CAAC,IAAI,CAAC,MAAM;QACjC,IAAI,IAAI,CAAC,aAAa,EACpB3B,MAAM;aACD;YACL,MAAM4B,YAAYF,KAAK,GAAG;YAC1B,MACE,CAAC,IAAI,CAAC,MAAM,CAAE,MAAM,MACpBA,KAAK,GAAG,KAAKE,YAAYtB,uBAEzB,MAAM,IAAIuB,QAAQ,CAACC,UAAYC,WAAWD,SAAS;YAErD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAE,MAAM,IACtB9B,MACE,CAAC,sBAAsB,EAAEM,uBAAuB,mBAAmB,CAAC;QAG1E;QACA,MAAM,IAAI,CAAC,WAAW;QACtB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO;QAE/B,IAAI,IAAI,CAAC,UAAU,GAAG,GAAG;YACvB,IAAI,CAAC,aAAa,GAAGyB,WAAW;gBAC9B7B,aACE,CAAC,8BAA8B,EAAE,IAAI,CAAC,UAAU,CAAC,kDAAkD,CAAC;gBAEtG,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,KAAO;YAC3B,GAAG,IAAI,CAAC,UAAU;YAClB,IACE,AACA,cADA,OAAQ,IAAI,CAAC,aAAa,CAA4B,KAAK,EAG1D,IAAI,CAAC,aAAa,CAA2B,KAAK;QAEvD;IACF;IAEA,OAAmC;QACjC,IAAI,CAAC,IAAI,CAAC,WAAW,EACnB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY;QAEtC,OAAO,IAAI,CAAC,WAAW;IACzB;IAEA,MAAc,eAA2C;QACvD,IAAI,CAAC,OAAO,GAAG;QACf,IAAI,IAAI,CAAC,aAAa,EAAE;YACtB8B,aAAa,IAAI,CAAC,aAAa;YAC/B,IAAI,CAAC,aAAa,GAAG;QACvB;QACA,MAAM,IAAI,CAAC,WAAW;QAEtB,IAAI;YACF,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,GACvB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC,CAACL;gBACxD3B,MAAM,CAAC,oDAAoD,EAAE2B,OAAO;YACtE;YAEF,MAAMM,wBAAwB,IAAI,CAAC,IAAI,CAAC,6BAA6B;YACrE,MAAM,GAAGC,eAAe,GAAG,MAAML,QAAQ,GAAG,CAAC;gBAC3C,IAAI,CAAC,cAAc;gBACnBI;aACD;YAED,MAAME,YAAY,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,MAAM,IAAIA,WAAW,WAAW;gBACvC,IAAI,CAAC,mBAAmB,GAAG;oBACzB,GAAGA,UAAU,SAAS;oBACtB,YAAYA,UAAU,UAAU;gBAClC;gBACAD,eAAe,UAAU,GAAGE,4CAAAA,cAAAA,CAAAA,QAAuB,CACjD,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAACD,UAAU,SAAS,GAChDA,UAAU,SAAS,CAAC,QAAQ,EAC5BA,UAAU,UAAU;gBAEtBnC,MAAM;YACR;YACA,IAAI,CAAC,cAAc,GAAGkC;YACtB,MAAMG,UAAUX,KAAK,GAAG;YACxB,MAAMP,SAAS,MAAM,IAAI,CAAC,cAAc,CAACkB;YACzC,IAAI,CAAC,WAAW,GAAG,IAAI3B,kBACrBS,QACA,IAAI,CAAC,IAAI,CAAC,aAAa,CAACA,SACxB,IAAM,IAAI,CAAC,MAAM,CAAC,OAAO,IACzB,IAAI,CAAC,IAAI,CAAC,UAAU;YAEtB,OAAO,IAAI,CAAC,WAAW;QACzB,SAAU;YACR,IAAI,IAAI,CAAC,MAAM,EACb,IAAI;gBACF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI;YACxB,EAAE,OAAOQ,OAAO;gBACd3B,MAAM,CAAC,6BAA6B,EAAE2B,OAAO;YAC/C;YAEF3B,MACE,CAAC,yBAAyB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,oCAAoC,CAAC;YAEtF,IAAI,CAAC,IAAI,CAAC,SAAS;QACrB;IACF;IAEA,MAAc,eAAeqC,OAAe,EAAgC;QAC1E1B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,EACnC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,MAAM,IAAI,CAAC,cAAc;YACzB,IAAI,CAAC,cAAc,GAAG;QACxB;QACA,MAAM,IAAI,CAAC,qBAAqB;QAEhC,MAAM2B,gBAAgB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAACC;YACrC5B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO4B,MAAM,SAAS,EAAE;YACxB,OAAO;gBAAE,GAAGA,MAAM,SAAS;gBAAE,YAAYA,MAAM,UAAU;YAAC;QAC5D;QAEA,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC7B,MAAML,iBAAiB,IAAI,CAAC,cAAc;YAG1C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CACjDA,eAAe,UAAU,CAAC,MAAM,EAChCA,eAAe,UAAU,CAAC,UAAU;YAEtCA,eAAe,UAAU,GAAGE,4CAAAA,cAAAA,CAAAA,QAAuB,CACjD,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,mBAAmB,GACrD,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EACjC,IAAI,CAAC,mBAAmB,CAAC,UAAU;QAEvC;QAEA,MAAMI,SAAS;eAAIF;YAAe,IAAI,CAAC,mBAAmB;SAAC;QAC3D,IAAIE,OAAO,MAAM,GAAGhC,uBAClBN,aACE,CAAC,mBAAmB,EAAEsC,OAAO,MAAM,CAAC,oBAAoB,EAAEhC,sBAAsB,wJAAwJ,CAAC;QAG7OR,MACE,CAAC,UAAU,EAAEwC,OAAO,MAAM,CAAC,iCAAiC,EAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,qBAAqB,CAAC;QAE/G,MAAMC,WAAwC;YAC5C,WAAW,IAAI,CAAC,SAAS;YACzBJ;YACA,UAAU;gBAAE,GAAG,IAAI,CAAC,cAAc,CAAE,QAAQ;YAAC;YAC7C,0BAA0B,IAAI,CAAC,cAAc,CAAE,wBAAwB;QACzE;QACA,OAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,CAACG,QAAQC;IACtC;IAGA,MAAM,UAAyB;QAC7B,IAAI,IAAI,CAAC,QAAQ,EAAE;QACnB,IAAI;YACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW;iBACjB,IAAI,CAAC,IAAI,CAAC,OAAO,EACtB,MAAM,IAAI,CAAC,IAAI;QAEnB,SAAU;YACR,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO;iBACzB;gBACL,IAAI,CAAC,MAAM,CAAC,OAAO;gBACnB,IAAI,CAAC,IAAI,CAAC,UAAU;YACtB;YACA,IAAI,CAAC,MAAM,GAAG,EAAE;YAChB,IAAI,CAAC,cAAc,CAAC,KAAK;YACzB,IAAI,CAAC,WAAW,GAAG;YACnB,IAAI,CAAC,QAAQ,GAAG;QAClB;IACF;IAEA,MAAc,wBAAuC;QACnD,MAAMC,eAAe,IAAI,CAAC,UAAU,CAClC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAACH,QAAU,CAACA,MAAM,SAAS;QAEhD,MAAMI,iBAAiBD,aAAa,MAAM,CACxC,CAACH,QAAU,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAACA,MAAM,GAAG;QAG/C,IACE,IAAIK,QAAQ,GACZA,QAAQD,eAAe,MAAM,EAC7BC,SAASnC,kBACT;YACA,MAAMoC,QAAQF,eAAe,KAAK,CAACC,OAAOA,QAAQnC;YAClD,MAAMqC,UAAU,IAAI,CAAC,MAAM,GACvB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAACD,SACzBA,MAAM,GAAG,CAAC,CAACN;gBACT5B,IAAAA,sBAAAA,MAAAA,AAAAA,EACEY,eAAegB,MAAM,GAAG,GACxB;gBAEF,OAAOA,MAAM,GAAG;YAClB;YACJ5B,IAAAA,sBAAAA,MAAAA,AAAAA,EACEmC,QAAQ,MAAM,KAAKD,MAAM,MAAM,EAC/B;YAEF,MAAME,mBAAmB,MAAMlB,QAAQ,GAAG,CACxCiB,QAAQ,GAAG,CAAC,CAACE,UACXC,AAAAA,IAAAA,mDAAAA,+BAAAA,AAAAA,EAAgCD,SAAS;oBACvC,cAAc,IAAI,CAAC,sBAAsB;gBAC3C;YAGJ,IAAK,IAAIE,QAAQ,GAAGA,QAAQL,MAAM,MAAM,EAAEK,QACxC,IAAI,CAAC,cAAc,CAAC,GAAG,CACrBL,KAAK,CAACK,MAAM,CAAC,GAAG,EAChB,IAAI,CAAC,MAAM,CAAC,YAAY,CACtBH,gBAAgB,CAACG,MAAM,EACvBL,KAAK,CAACK,MAAM,CAAC,UAAU;QAI/B;QAEA,KAAK,MAAMX,SAAS,IAAI,CAAC,MAAM,CAC7BA,MAAM,SAAS,KAAK,IAAI,CAAC,cAAc,CAAC,GAAG,CAACA,MAAM,GAAG;QAEvD,IAAII,eAAe,MAAM,GAAG,GAC1B3C,MAAM,CAAC,sBAAsB,EAAE2C,eAAe,MAAM,CAAC,cAAc,CAAC;IAExE;IAEA,MAAc,cAA6B;QACzC,IAAI;YACF,IAAI,IAAI,CAAC,MAAM,EAAE;gBACf,MAAMJ,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM;gBAChC,IAAI,CAACA,OAAO;gBACZ,IAAIhB,eAAegB,MAAM,GAAG,GAAG;oBAC7B,MAAMY,kBAAkB,MAAMF,AAAAA,IAAAA,mDAAAA,+BAAAA,AAAAA,EAC5BV,MAAM,GAAG,EACT;wBAAE,cAAc,IAAI,CAAC,sBAAsB;oBAAC;oBAE9C,MAAMa,YAAY,IAAI,CAAC,MAAM,CAAC,YAAY,CACxCD,iBACAZ,MAAM,UAAU;oBAElB,IAAI,CAAC,SAAS,CAAC;wBACb,KAAKa,UAAU,IAAI;wBACnB,YAAYb,MAAM,UAAU;wBAC5Ba;oBACF;gBACF,OACE,IAAI,CAAC,SAAS,CAACb;gBAEjB;YACF;YACA,MAAMY,kBAAkB,MAAMF,AAAAA,IAAAA,mDAAAA,+BAAAA,AAAAA,EAC5B,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,IACpC;gBAAE,cAAc,IAAI,CAAC,sBAAsB;YAAC;YAE9C,MAAMG,YAAY,IAAI,CAAC,MAAM,CAAC,YAAY,CAACD,iBAAiBzB,KAAK,GAAG;YACpE,IAAI,CAAC,SAAS,CAAC;gBACb,KAAK0B,UAAU,IAAI;gBACnB,YAAYA,UAAU,UAAU;gBAChCA;YACF;QACF,EAAE,OAAOzB,OAAO;YACd3B,MAAM,CAAC,qCAAqC,EAAE2B,OAAO;QACvD;IACF;IAEA,MAAc,UAAyB;QACrC,MAAO,CAAC,IAAI,CAAC,OAAO,CAAE;YACpB,MAAM0B,YAAY3B,KAAK,GAAG;YAC1B,MAAM,IAAI,CAAC,WAAW;YACtB,MAAO,CAAC,IAAI,CAAC,OAAO,IAAIA,KAAK,GAAG,KAAK2B,YAAY,IAAI,CAAC,UAAU,CAC9D,MAAM,IAAIxB,QAAQ,CAACC,UAAYC,WAAWD,SAAS;QAEvD;IACF;IAEQ,UAAUS,KAAqC,EAAQ;QAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CAACA;QACjB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;YACvC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM;YACzC,IAAI,CAAC,MAAM,CAAC,WAAW,CACrB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAACe,gBACnBA,cAAc,SAAS,GAAG;oBAACA,cAAc,SAAS;iBAAC,GAAG,EAAE;YAG5DtD,MAAM,CAAC,wBAAwB,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;QAC9D;IACF;IAEQ,WAAWwC,MAAuB,EAAmB;QAC3D,IAAIA,OAAO,MAAM,IAAI,GAAG,OAAOA;QAC/B,MAAMe,gBAAgB,IAAIC,MAAMhB,OAAO,MAAM,EAAE,IAAI,CAAC;QACpDe,aAAa,CAAC,EAAE,GAAG;QACnB,IAAK,IAAIL,QAAQ,GAAGA,QAAQV,OAAO,MAAM,EAAEU,QACzC,IAAIV,MAAM,CAACU,MAAM,CAAC,GAAG,KAAKV,MAAM,CAACU,QAAQ,EAAE,CAAC,GAAG,EAC7CK,aAAa,CAACL,MAAM,GAAG;QAG3BK,aAAa,CAACf,OAAO,MAAM,GAAG,EAAE,GAAG;QAEnC,IAAIiB,SAA0B,EAAE;QAChC,IAAIC,gBAAgB;QACpB,IAAK,IAAIR,QAAQ,GAAGA,QAAQV,OAAO,MAAM,EAAEU,QACzC,IAAIK,aAAa,CAACL,MAAM,EAAE;YACxBO,OAAO,IAAI,CAACjB,MAAM,CAACU,MAAM;YACzBQ,gBAAgB;QAClB,OAAO,IAAIA,gBAAgB,MAAM,GAAG;YAClCD,OAAO,IAAI,CAACjB,MAAM,CAACU,MAAM;YACzBQ;QACF,OACEA;QAIJ,IAAID,OAAO,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;YAClC,MAAME,OAAOF,OAAO,MAAM,GAAG,IAAI,CAAC,SAAS;YAC3C,MAAMG,UAA2B,EAAE;YACnC,IAAK,IAAIV,QAAQ,GAAGA,QAAQ,IAAI,CAAC,SAAS,EAAEA,QAC1CU,QAAQ,IAAI,CAACH,MAAM,CAACI,KAAK,KAAK,CAACX,QAAQS,MAAM;YAE/CC,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,EAAE,GAAGH,MAAM,CAACA,OAAO,MAAM,GAAG,EAAE;YACvDA,SAASG;QACX;QACA,OAAOH;IACT;IAEQ,WAAWjB,MAAuB,EAAmB;QAC3D,MAAMsB,OAAO,IAAIC;QACjB,MAAMN,SAA0B,EAAE;QAClC,KAAK,MAAMlB,SAASC,OAClB,IAAI,CAACsB,KAAK,GAAG,CAACvB,MAAM,GAAG,GAAG;YACxBuB,KAAK,GAAG,CAACvB,MAAM,GAAG;YAClBkB,OAAO,IAAI,CAAClB;QACd;QAEF,OAAOkB;IACT;IA3WA,YACmBO,IAAoB,EACrCC,GAAsB,CACtB;;QAvBF,uBAAQ,UAAR;QACA,uBAAQ,UAAR;QACA,uBAAQ,iBAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,YAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,kBAAR;QACA,uBAAQ,uBAAR;QACA,uBAAQ,iBAAR;QACA,uBAAQ,kBAAR;QACA,uBAAQ,kBAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,aAAR;QACA,uBAAiB,cAAjB;QACA,uBAAiB,aAAjB;QACA,uBAAiB,cAAjB;QACA,uBAAiB,0BAAjB;QACA,uBAAiB,UAAjB;aAGmBD,IAAI,GAAJA;aArBX,MAAM,GAAoB,EAAE;aAC5B,MAAM,GAA6B;aACnC,aAAa,GAAG;aAChB,OAAO,GAAG;aACV,QAAQ,GAAG;aACX,WAAW,GAAyB;aACpC,WAAW,GAAsC;aACjD,cAAc,GAAqB;aACnC,mBAAmB,GAA8B;aACjD,aAAa,GAAyC;aACtD,cAAc,GAAyB;aACvC,cAAc,GAAG,IAAIE;aACrB,WAAW,GAA6B;aACxC,SAAS,GAAG;QAWlB,IAAI,CAAC,UAAU,GAAGL,KAAK,GAAG,CACxBzD,iBACA6D,KAAK,cAAc9D;QAErB,IAAI,CAAC,SAAS,GAAG0D,KAAK,GAAG,CAAC,GAAGI,KAAK,aAAa5D;QAC/C,IAAI,CAAC,UAAU,GAAG4D,KAAK,cAAc1D;QACrC,IAAI,CAAC,sBAAsB,GAAGyD,KAAK,sBAAsB,IAAI;QAC7D,IAAI,CAAC,MAAM,GACTA,KAAK,uBAAuB,IAAI,IAAIG,mCAAAA,yBAAyBA;IACjE;AA+VF;AAGO,SAASC,+BACdjD,MAA2B;IAE3BR,IAAAA,sBAAAA,MAAAA,AAAAA,EACEQ,AAAgB,8BAAhBA,OAAO,IAAI,IAAkCA,AAAmB,MAAnBA,OAAO,OAAO,EAC3D;IAEFR,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOQ,OAAO,MAAM,CAAC,MAAM,GAAG,GAAG;IACjCR,IAAAA,sBAAAA,MAAAA,AAAAA,EACE0D,OAAO,QAAQ,CAAClD,OAAO,QAAQ,CAAC,KAAK,KAAKA,OAAO,QAAQ,CAAC,KAAK,GAAG,GAClE;IAEFR,IAAAA,sBAAAA,MAAAA,AAAAA,EACE0D,OAAO,QAAQ,CAAClD,OAAO,QAAQ,CAAC,MAAM,KAAKA,OAAO,QAAQ,CAAC,MAAM,GAAG,GACpE;IAEFR,IAAAA,sBAAAA,MAAAA,AAAAA,EACE0D,OAAO,QAAQ,CAAClD,OAAO,wBAAwB,KAC7CA,OAAO,wBAAwB,GAAG,GACpC;IAEF,MAAMmD,qBAAqBnD,OAAO,MAAM,CAAC,GAAG,CAAC,CAACoB,QAC5CH,4CAAAA,cAAAA,CAAAA,QAAuB,CAACG,MAAM,IAAI,EAAEA,MAAM,QAAQ,EAAEA,MAAM,UAAU;IAEtE,OAAO;QACL,YAAY+B,kBAAkB,CAACA,mBAAmB,MAAM,GAAG,EAAE;QAC7DA;QACA,UAAU;YAAE,GAAGnD,OAAO,QAAQ;QAAC;QAC/B,0BAA0BA,OAAO,wBAAwB;IAC3D;AACF"}

package/dist/lib/agent/utils.js

@@ -33,18 +33,22 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
+    createScreenshotBoundUIContext: ()=>createScreenshotBoundUIContext,
     getMidsceneVersion: ()=>getMidsceneVersion,
     getReportFileName: ()=>getReportFileName,
     matchElementFromPlan: ()=>matchElementFromPlan,
-    parsePrompt: ()=>parsePrompt,
     matchElementFromCache: ()=>matchElementFromCache,
     commonContextParser: ()=>commonContextParser,
+    normalizeFilePaths: ()=>normalizeFilePaths,
+    parsePrompt: ()=>parsePrompt,
     transformLogicalElementToScreenshot: ()=>transformLogicalElementToScreenshot,
     ifPlanLocateParamIsBbox: ()=>ifPlanLocateParamIsBbox,
     printReportMsg: ()=>printReportMsg,
     transformLogicalRectToScreenshotRect: ()=>transformLogicalRectToScreenshotRect
 });
 const external_screenshot_item_js_namespaceObject = require("../screenshot-item.js");
+const external_node_fs_namespaceObject = require("node:fs");
+const external_node_path_namespaceObject = require("node:path");
 const external_utils_js_namespaceObject = require("../utils.js");
 const env_namespaceObject = require("@aiscene/shared/env");
 const extractor_namespaceObject = require("@aiscene/shared/extractor");
@@ -54,6 +58,19 @@ const utils_namespaceObject = require("@aiscene/shared/utils");
 const external_dayjs_namespaceObject = require("dayjs");
 var external_dayjs_default = /*#__PURE__*/ __webpack_require__.n(external_dayjs_namespaceObject);
 const external_task_cache_js_namespaceObject = require("./task-cache.js");
+const agentDebug = (0, logger_namespaceObject.getDebug)('agent');
+const screenshotDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i;
+const inferBase64ImageFormat = (base64Body)=>{
+    if (base64Body.startsWith('iVBORw0KGgo')) return 'png';
+    return 'jpeg';
+};
+const normalizeScreenshotBase64 = (screenshotBase64)=>{
+    const trimmedBase64 = screenshotBase64.trim();
+    if (screenshotDataUrlPattern.test(trimmedBase64)) return trimmedBase64;
+    const base64Body = trimmedBase64.replace(/\s/g, '');
+    (0, utils_namespaceObject.assert)(base64Body, 'screenshotBase64 must include image data');
+    return (0, img_namespaceObject.createImgBase64ByFormat)(inferBase64ImageFormat(base64Body), base64Body);
+};
 async function commonContextParser(interfaceInstance, _opt) {
     const debug = (0, logger_namespaceObject.getDebug)('commonContextParser');
     (0, utils_namespaceObject.assert)(interfaceInstance, 'interfaceInstance is required');
@@ -122,6 +139,20 @@ async function commonContextParser(interfaceInstance, _opt) {
         shrunkShotToLogicalRatio
     };
 }
+async function createScreenshotBoundUIContext(screenshotBase64, opt) {
+    const normalizedScreenshotBase64 = normalizeScreenshotBase64(screenshotBase64);
+    const actualScreenshotSize = await (0, img_namespaceObject.imageInfoOfBase64)(normalizedScreenshotBase64);
+    if (opt.screenshotSize && (opt.screenshotSize.width !== actualScreenshotSize.width || opt.screenshotSize.height !== actualScreenshotSize.height)) agentDebug('describeElementAtPoint screenshotSize mismatch, use actual size', {
+        provided: opt.screenshotSize,
+        actual: actualScreenshotSize
+    });
+    return {
+        screenshot: external_screenshot_item_js_namespaceObject.ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),
+        shotSize: actualScreenshotSize,
+        shrunkShotToLogicalRatio: 1,
+        _isFrozen: true
+    };
+}
 function getReportFileName(tag = 'web') {
     const reportTagName = env_namespaceObject.globalConfigManager.getEnvConfigValue(env_namespaceObject.MIDSCENE_REPORT_TAG_NAME);
     const dateTimeInFileName = external_dayjs_default()().format('YYYY-MM-DD_HH-mm-ss');
@@ -170,7 +201,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"9.0.5";
+const getMidsceneVersion = ()=>"9.0.8";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,
@@ -184,6 +215,15 @@ const parsePrompt = (prompt)=>{
         } : void 0
     };
 };
+function normalizeFilePaths(files, options = {}) {
+    const { fileExists = external_node_fs_namespaceObject.existsSync, isInBrowser = utils_namespaceObject.ifInBrowser, resolvePath = external_node_path_namespaceObject.resolve, cwd = process.cwd() } = options;
+    if (isInBrowser) throw new Error('File chooser is not supported in browser environment');
+    return files.map((file)=>{
+        const absolutePath = resolvePath(file);
+        if (!fileExists(absolutePath)) throw new Error(`File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`);
+        return absolutePath;
+    });
+}
 const transformLogicalElementToScreenshot = (element, shrunkShotToLogicalRatio)=>{
     if (1 === shrunkShotToLogicalRatio) return element;
     return {
@@ -212,22 +252,26 @@ const transformLogicalRectToScreenshotRect = (rect, shrunkShotToLogicalRatio)=>{
     };
 };
 exports.commonContextParser = __webpack_exports__.commonContextParser;
+exports.createScreenshotBoundUIContext = __webpack_exports__.createScreenshotBoundUIContext;
 exports.getMidsceneVersion = __webpack_exports__.getMidsceneVersion;
 exports.getReportFileName = __webpack_exports__.getReportFileName;
 exports.ifPlanLocateParamIsBbox = __webpack_exports__.ifPlanLocateParamIsBbox;
 exports.matchElementFromCache = __webpack_exports__.matchElementFromCache;
 exports.matchElementFromPlan = __webpack_exports__.matchElementFromPlan;
+exports.normalizeFilePaths = __webpack_exports__.normalizeFilePaths;
 exports.parsePrompt = __webpack_exports__.parsePrompt;
 exports.printReportMsg = __webpack_exports__.printReportMsg;
 exports.transformLogicalElementToScreenshot = __webpack_exports__.transformLogicalElementToScreenshot;
 exports.transformLogicalRectToScreenshotRect = __webpack_exports__.transformLogicalRectToScreenshotRect;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
     "commonContextParser",
+    "createScreenshotBoundUIContext",
     "getMidsceneVersion",
     "getReportFileName",
     "ifPlanLocateParamIsBbox",
     "matchElementFromCache",
     "matchElementFromPlan",
+    "normalizeFilePaths",
     "parsePrompt",
     "printReportMsg",
     "transformLogicalElementToScreenshot",