@midscene/playground 1.9.6 → 1.9.7

package/dist/es/server.mjs

@@ -1,13 +1,16 @@
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname, join, resolve as external_node_path_resolve } from "node:path";
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve as external_node_path_resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { ReportActionDump, runConnectivityTest } from "@midscene/core";
 import { getTmpDir, sleep } from "@midscene/core/utils";
+import { getMidsceneRunSubDir } from "@midscene/shared/common";
 import { PLAYGROUND_SERVER_PORT } from "@midscene/shared/constants";
 import { globalModelConfigManager, overrideAIConfig } from "@midscene/shared/env";
 import { generateElementByPoint } from "@midscene/shared/extractor";
-import { compositePointMarkerImg } from "@midscene/shared/img";
+import { annotateRects, compositePointMarkerImg, imageInfoOfBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
+import { buildMidsceneRecorderActionSummary, buildMidsceneRecorderReplayInstruction } from "@midscene/shared/recorder";
 import { uuid as utils_uuid } from "@midscene/shared/utils";
 import express from "express";
 import { executeAction, formatErrorMessage } from "./common.mjs";
@@ -27,6 +30,216 @@ function _define_property(obj, key, value) {
 }
 const defaultPort = PLAYGROUND_SERVER_PORT;
 const RECORDER_CAPTURE_AFTER_INTERACT_DELAY_MS = 250;
+const RECORDER_AI_DESCRIBE_AFTER_INTERACT_TIMEOUT_MS = 30000;
+const RECORDER_AI_DESCRIBE_SCREENSHOT_DUMP_DIR = 'recorder-ai-describe-screenshots';
+function extractBase64Payload(value) {
+    const dataUrlMatch = value.match(/^data:([^;,]+)?(?:;[^,]*)?,([\s\S]*)$/);
+    if (dataUrlMatch) return {
+        mimeType: dataUrlMatch[1],
+        base64: dataUrlMatch[2] || ''
+    };
+    return {
+        base64: value
+    };
+}
+function estimateBase64Bytes(value) {
+    if (!value) return;
+    const { base64 } = extractBase64Payload(value);
+    const padding = base64.endsWith('==') ? 2 : base64.endsWith('=') ? 1 : 0;
+    return Math.max(0, Math.floor(3 * base64.length / 4) - padding);
+}
+function buildRecorderRawPayloadSummary(rawPayload) {
+    if (!rawPayload) return;
+    const allowedKeys = [
+        'actionType',
+        'x',
+        'y',
+        'endX',
+        'endY',
+        'duration',
+        'direction',
+        'scrollType',
+        'distance',
+        'keyName',
+        'mode'
+    ];
+    const summary = {};
+    for (const key of allowedKeys)if (void 0 !== rawPayload[key]) summary[key] = rawPayload[key];
+    const value = rawPayload.value;
+    if ('string' == typeof value) summary.valueLength = value.length;
+    return Object.keys(summary).length > 0 ? summary : void 0;
+}
+function buildRecorderEventSummary(event) {
+    return {
+        hashId: event.hashId,
+        mergedHashIds: event.mergedHashIds,
+        type: event.type,
+        source: event.source,
+        actionType: event.actionType,
+        timestamp: event.timestamp,
+        url: event.url,
+        title: event.title,
+        valueLength: 'string' == typeof event.value ? event.value.length : void 0,
+        rawPayloadSummary: buildRecorderRawPayloadSummary(event.rawPayload),
+        elementRect: event.elementRect,
+        pageInfo: event.pageInfo
+    };
+}
+function shouldVerifyRecorderAiDescribeEvent(event) {
+    return 'scroll' !== event.type;
+}
+function formatRecorderAiDescribeScreenshotPathParts() {
+    const date = new Date();
+    const localDate = Number.isNaN(date.getTime()) ? new Date() : date;
+    const pad = (value, length = 2)=>String(value).padStart(length, '0');
+    const datePart = [
+        localDate.getFullYear(),
+        pad(localDate.getMonth() + 1),
+        pad(localDate.getDate())
+    ].join('-');
+    const timePart = [
+        pad(localDate.getHours()),
+        pad(localDate.getMinutes()),
+        pad(localDate.getSeconds()),
+        pad(localDate.getMilliseconds(), 3)
+    ].join('-');
+    const hourPart = timePart.slice(0, 2) || 'unknown-hour';
+    return {
+        datePart,
+        hourPart,
+        filePrefix: `${timePart}_${randomUUID()}`
+    };
+}
+function assertPathInsideDirectory(rootDir, targetPath) {
+    const resolvedRoot = external_node_path_resolve(rootDir);
+    const resolvedTarget = external_node_path_resolve(targetPath);
+    if (resolvedTarget !== resolvedRoot && !resolvedTarget.startsWith(`${resolvedRoot}${sep}`)) throw new Error(`Refusing to write recorder dump outside ${resolvedRoot}`);
+    return resolvedTarget;
+}
+function writeRecorderAiDescribeScreenshot(imageBase64, suffix) {
+    const { base64, mimeType } = extractBase64Payload(imageBase64);
+    const bytes = Buffer.from(base64, 'base64');
+    const sha256 = createHash('sha256').update(bytes).digest('hex');
+    const extension = mimeType?.includes('jpeg') ? 'jpg' : 'png';
+    const pathParts = formatRecorderAiDescribeScreenshotPathParts();
+    const dumpRoot = external_node_path_resolve(getMidsceneRunSubDir('dump'), RECORDER_AI_DESCRIBE_SCREENSHOT_DUMP_DIR);
+    const dumpDir = assertPathInsideDirectory(dumpRoot, join(dumpRoot, pathParts.datePart, pathParts.hourPart));
+    mkdirSync(dumpDir, {
+        recursive: true
+    });
+    const safeSuffix = 'annotated' === suffix ? 'annotated' : 'raw';
+    const fileName = `${pathParts.filePrefix}_${safeSuffix}.${extension}`;
+    const filePath = assertPathInsideDirectory(dumpRoot, join(dumpDir, fileName));
+    if (!existsSync(filePath)) writeFileSync(filePath, bytes);
+    return {
+        path: filePath,
+        sha256,
+        bytes: bytes.byteLength,
+        mimeType
+    };
+}
+function calculateRecorderScreenshotAnnotation(event, imageSize, verifyResult) {
+    const pageWidth = event.pageInfo?.width;
+    const pageHeight = event.pageInfo?.height;
+    const scaleX = pageWidth ? imageSize.width / pageWidth : void 0;
+    const scaleY = pageHeight ? imageSize.height / pageHeight : void 0;
+    const mapX = (value)=>scaleX ? value * scaleX : value;
+    const mapY = (value)=>scaleY ? value * scaleY : value;
+    const logicalPoint = 'number' == typeof event.elementRect?.x && 'number' == typeof event.elementRect?.y ? [
+        event.elementRect.x,
+        event.elementRect.y
+    ] : void 0;
+    const locateRect = verifyResult?.rect;
+    const sourceTargetRect = 'number' == typeof event.elementRect?.left && 'number' == typeof event.elementRect?.top && 'number' == typeof event.elementRect?.width && 'number' == typeof event.elementRect?.height && event.elementRect.width > 0 && event.elementRect.height > 0 ? {
+        left: mapX(event.elementRect.left),
+        top: mapY(event.elementRect.top),
+        width: mapX(event.elementRect.width),
+        height: mapY(event.elementRect.height)
+    } : void 0;
+    if (!logicalPoint && !sourceTargetRect && !locateRect) return;
+    const screenshotPoint = logicalPoint && scaleX && scaleY ? [
+        logicalPoint[0] * scaleX,
+        logicalPoint[1] * scaleY
+    ] : void 0;
+    const pointTargetRect = screenshotPoint && !sourceTargetRect ? {
+        left: Math.max(0, screenshotPoint[0] - 6),
+        top: Math.max(0, screenshotPoint[1] - 6),
+        width: 12,
+        height: 12
+    } : void 0;
+    const center = verifyResult?.center || (locateRect ? [
+        locateRect.left + locateRect.width / 2,
+        locateRect.top + locateRect.height / 2
+    ] : void 0);
+    const centerDelta = screenshotPoint && center ? {
+        x: center[0] - screenshotPoint[0],
+        y: center[1] - screenshotPoint[1],
+        distance: Math.hypot(center[0] - screenshotPoint[0], center[1] - screenshotPoint[1])
+    } : void 0;
+    const distanceOutsideRect = screenshotPoint && locateRect ? (()=>{
+        const right = locateRect.left + locateRect.width;
+        const bottom = locateRect.top + locateRect.height;
+        const x = screenshotPoint[0] < locateRect.left ? locateRect.left - screenshotPoint[0] : screenshotPoint[0] > right ? screenshotPoint[0] - right : 0;
+        const y = screenshotPoint[1] < locateRect.top ? locateRect.top - screenshotPoint[1] : screenshotPoint[1] > bottom ? screenshotPoint[1] - bottom : 0;
+        return {
+            x,
+            y,
+            distance: Math.hypot(x, y)
+        };
+    })() : void 0;
+    return {
+        inputPoint: screenshotPoint ? {
+            logical: logicalPoint,
+            screenshot: screenshotPoint
+        } : void 0,
+        sourceTargetRect: sourceTargetRect || pointTargetRect,
+        locateRect,
+        centerDelta,
+        distanceOutsideRect
+    };
+}
+async function persistRecorderAiDescribeScreenshot(event, eventScreenshot, verifyResult) {
+    if (!eventScreenshot) return;
+    const screenshotRef = writeRecorderAiDescribeScreenshot(eventScreenshot, 'raw');
+    let annotatedScreenshotRef;
+    let screenshotAnnotation;
+    let annotatedScreenshotPersistError;
+    try {
+        const imageSize = await imageInfoOfBase64(eventScreenshot);
+        screenshotAnnotation = calculateRecorderScreenshotAnnotation(event, imageSize, verifyResult);
+        const annotatedRects = [
+            screenshotAnnotation?.sourceTargetRect,
+            screenshotAnnotation?.locateRect
+        ].filter((rect)=>Boolean(rect));
+        if (annotatedRects.length > 0) {
+            const annotatedScreenshot = await annotateRects(eventScreenshot, annotatedRects);
+            annotatedScreenshotRef = writeRecorderAiDescribeScreenshot(annotatedScreenshot, 'annotated');
+        }
+    } catch (error) {
+        annotatedScreenshotPersistError = error instanceof Error ? error.message : String(error);
+    }
+    return {
+        screenshotRef,
+        annotatedScreenshotRef,
+        screenshotAnnotation,
+        annotatedScreenshotPersistError
+    };
+}
+function createRecorderAiDescribeTraceBase(event, eventScreenshot) {
+    return {
+        traceId: `${event.hashId || 'recorder-event'}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`,
+        eventHashId: event.hashId,
+        eventType: event.type,
+        actionType: event.actionType,
+        eventSummary: buildRecorderEventSummary(event),
+        point: 'number' == typeof event.elementRect?.x && 'number' == typeof event.elementRect?.y ? [
+            event.elementRect.x,
+            event.elementRect.y
+        ] : void 0,
+        pageInfo: event.pageInfo,
+        screenshotBytes: estimateBase64Bytes(eventScreenshot)
+    };
+}
 function serializeAiConfigSignature(aiConfig) {
     return JSON.stringify(Object.entries(aiConfig).sort(([leftKey], [rightKey])=>leftKey.localeCompare(rightKey)));
 }
@@ -190,6 +403,74 @@ function buildInteractParams(actionType, body) {
     const { actionType: _omit, ...passthrough } = body;
     return passthrough;
 }
+function getRecorderSemanticActionType(actionType) {
+    switch(actionType){
+        case 'Tap':
+        case 'DoubleClick':
+        case 'LongPress':
+        case 'RightClick':
+            return 'click';
+        case 'Swipe':
+        case 'DragAndDrop':
+            return 'drag';
+        case 'Input':
+            return 'input';
+        case 'KeyboardPress':
+            return 'keydown';
+        case 'Scroll':
+            return 'scroll';
+        case 'GoBack':
+        case 'GoForward':
+        case 'Reload':
+        case 'Stop':
+        case 'InitialNavigation':
+        case 'NavigationChanged':
+            return 'navigation';
+        default:
+            return 'click';
+    }
+}
+function buildRecorderSemanticAction(actionType, payload, url) {
+    return {
+        type: getRecorderSemanticActionType(actionType),
+        actionType,
+        value: 'string' == typeof payload.value ? payload.value : 'string' == typeof payload.keyName ? payload.keyName : void 0,
+        url
+    };
+}
+function buildReadyRecorderSemantic(source, event, elementDescription, confidence, extra) {
+    return {
+        source,
+        status: 'ready',
+        elementDescription,
+        replayInstruction: buildMidsceneRecorderReplayInstruction(event, elementDescription),
+        actionSummary: buildMidsceneRecorderActionSummary(event, elementDescription),
+        confidence,
+        ...extra
+    };
+}
+function buildFailedAiDescribeRecorderSemantic(error, extra) {
+    return {
+        source: 'aiDescribe',
+        status: 'failed',
+        error: error instanceof Error ? error.message : String(error),
+        ...extra
+    };
+}
+function withTimeout(promise, timeoutMs, message) {
+    let timeout;
+    const timeoutPromise = new Promise((_resolve, reject)=>{
+        timeout = setTimeout(()=>{
+            reject(new Error(message));
+        }, timeoutMs);
+    });
+    return Promise.race([
+        promise,
+        timeoutPromise
+    ]).finally(()=>{
+        if (timeout) clearTimeout(timeout);
+    });
+}
 function createManualExecutorContext(actionType, param) {
     const task = {
         type: 'Action Space',
@@ -393,9 +674,14 @@ class PlaygroundServer {
     resetRecorderState() {
         this._recorderSessionId = null;
         this._recorderEvents = [];
+        this._recorderEventQueue = Promise.resolve();
+        this._studioPreviewRecorderLastTargetPoint = void 0;
         this._studioPreviewRecorderLastScreenshot = void 0;
         this._studioPreviewRecorderLastPageState = void 0;
     }
+    async waitForRecorderIdle() {
+        await this._recorderEventQueue;
+    }
     canRecordStudioPreviewInteractions() {
         const agent = this._activeConnection.agent;
         if (!agent) return false;
@@ -479,6 +765,13 @@ class PlaygroundServer {
             pageState: this._studioPreviewRecorderLastPageState || await this.getActiveRecorderPageState()
         };
     }
+    captureCachedRecorderSnapshotBeforeInteract() {
+        if (!this._recorderSessionId || !this._studioPreviewRecorderLastPageState || this._recorderPendingCaptures > 0) return;
+        return {
+            screenshot: this._mjpegHandler.getLastFrameBase64() || this._studioPreviewRecorderLastScreenshot,
+            pageState: this._studioPreviewRecorderLastPageState
+        };
+    }
     async startStudioPreviewRecorder(sessionId) {
         this._recorderSessionId = sessionId;
         this._studioPreviewRecorderLastScreenshot = await this.takeRecorderScreenshot();
@@ -487,7 +780,7 @@ class PlaygroundServer {
         const initialNavigationEvent = this.buildStudioPreviewInitialNavigationEvent(initialPageState, this._studioPreviewRecorderLastScreenshot);
         if (initialNavigationEvent) this._recorderEvents.push(initialNavigationEvent);
     }
-    async storeStudioPreviewRecorderEvent(payload, snapshotBefore) {
+    async storeStudioPreviewRecorderEvent(payload, snapshotBefore, agent) {
         if (!this._recorderSessionId) return;
         const before = snapshotBefore || await this.captureRecorderSnapshotBeforeInteract();
         const screenshotBefore = before?.screenshot;
@@ -512,19 +805,26 @@ class PlaygroundServer {
             this._studioPreviewRecorderLastPageState = pageStateAfter;
             return;
         }
-        this._recorderEvents.push(event);
         const navigationEvent = this.buildStudioPreviewNavigationChangeEvent(payload, before?.pageState, pageStateAfter, screenshotAfter);
-        if (navigationEvent) this._recorderEvents.push(navigationEvent);
         this._studioPreviewRecorderLastScreenshot = screenshotAfter;
         this._studioPreviewRecorderLastPageState = pageStateAfter;
+        this.queueStudioPreviewRecorderEventAppend(event, navigationEvent);
     }
     async buildStudioPreviewRecorderEvent(payload, pageState, screenshotBefore, screenshotAfter) {
         const actionType = 'string' == typeof payload.actionType ? payload.actionType : void 0;
         if (!actionType) return null;
         const { pageInfo, url, title } = pageState;
         const timestamp = Date.now();
-        const x = 'number' == typeof payload.x ? payload.x : void 0;
-        const y = 'number' == typeof payload.y ? payload.y : void 0;
+        const payloadX = 'number' == typeof payload.x ? payload.x : void 0;
+        const payloadY = 'number' == typeof payload.y ? payload.y : void 0;
+        const canReuseLastTargetPoint = 'Input' === actionType || 'KeyboardPress' === actionType;
+        const inheritedTargetPoint = canReuseLastTargetPoint && (void 0 === payloadX || void 0 === payloadY) ? this._studioPreviewRecorderLastTargetPoint : void 0;
+        const x = payloadX ?? inheritedTargetPoint?.x;
+        const y = payloadY ?? inheritedTargetPoint?.y;
+        if (void 0 !== payloadX && void 0 !== payloadY) this._studioPreviewRecorderLastTargetPoint = {
+            x: payloadX,
+            y: payloadY
+        };
         const endX = 'number' == typeof payload.endX ? payload.endX : void 0;
         const endY = 'number' == typeof payload.endY ? payload.endY : void 0;
         const dragDescription = void 0 !== x && void 0 !== y && void 0 !== endX && void 0 !== endY ? `${Math.round(x)},${Math.round(y)} -> ${Math.round(endX)},${Math.round(endY)}` : void 0;
@@ -542,8 +842,10 @@ class PlaygroundServer {
             screenshotBefore,
             screenshotAfter,
             screenshotWithBox: screenshotWithMarker,
-            descriptionLoading: true,
-            descriptionSource: void 0,
+            semantic: {
+                source: 'aiDescribe',
+                status: 'pending'
+            },
             timestamp,
             hashId: `studio-preview-${actionType}-${timestamp}-${Math.random().toString(36).slice(2, 8)}`
         };
@@ -621,17 +923,18 @@ class PlaygroundServer {
             case 'GoForward':
             case 'Reload':
             case 'Stop':
-                return {
-                    ...base,
-                    type: 'navigation',
-                    value: actionType,
-                    elementDescription: url || actionType,
-                    replayInstruction: 'Stop' === actionType ? 'Stop loading the current page.' : url ? `Wait for navigation to complete at \`${url}\`.` : `${actionType} in the browser.`,
-                    actionSummary: 'Stop' === actionType ? 'Stop page loading' : url ? `${actionType} to ${url}` : actionType,
-                    semanticConfidence: url ? 'high' : 'medium',
-                    descriptionLoading: false,
-                    descriptionSource: 'fallback'
-                };
+                {
+                    const elementDescription = url || actionType;
+                    const semanticEvent = buildRecorderSemanticAction(actionType, {
+                        value: actionType
+                    }, url);
+                    return {
+                        ...base,
+                        type: 'navigation',
+                        value: actionType,
+                        semantic: buildReadyRecorderSemantic('heuristic', semanticEvent, elementDescription, url ? 'high' : 'medium')
+                    };
+                }
             default:
                 return {
                     ...base,
@@ -640,6 +943,190 @@ class PlaygroundServer {
                 };
         }
     }
+    async enrichStudioPreviewRecorderEventWithAiDescribe(event, agent) {
+        const startedAt = new Date();
+        const startedAtMs = Date.now();
+        const eventScreenshot = this.getRecorderAiDescribeScreenshot(event);
+        const traceBase = createRecorderAiDescribeTraceBase(event, eventScreenshot);
+        const finishTrace = async (status, extra = {})=>{
+            let screenshotRef;
+            let annotatedScreenshotRef;
+            let screenshotAnnotation;
+            let screenshotPersistError;
+            let annotatedScreenshotPersistError;
+            if ('failed' === status) try {
+                const screenshotDump = await persistRecorderAiDescribeScreenshot(event, eventScreenshot, extra.verifyResult);
+                screenshotRef = screenshotDump?.screenshotRef;
+                annotatedScreenshotRef = screenshotDump?.annotatedScreenshotRef;
+                screenshotAnnotation = screenshotDump?.screenshotAnnotation;
+                annotatedScreenshotPersistError = screenshotDump?.annotatedScreenshotPersistError;
+            } catch (error) {
+                screenshotPersistError = error instanceof Error ? error.message : String(error);
+            }
+            return {
+                ...traceBase,
+                ...extra,
+                screenshotRef,
+                annotatedScreenshotRef,
+                screenshotAnnotation,
+                screenshotPersistError,
+                annotatedScreenshotPersistError,
+                status,
+                startedAt: startedAt.toISOString(),
+                durationMs: Date.now() - startedAtMs
+            };
+        };
+        if ('navigation' === event.type || 'setViewport' === event.type) {
+            const error = 'aiDescribe skipped because the event type does not target a UI element.';
+            const trace = await finishTrace('failed', {
+                error
+            });
+            debugInteract('recorder aiDescribe trace:', trace);
+            return {
+                event: {
+                    ...event,
+                    semantic: buildFailedAiDescribeRecorderSemantic(error)
+                },
+                trace
+            };
+        }
+        if ('function' != typeof agent?.describeElementAtPoint) {
+            const error = 'Active agent does not support describeElementAtPoint.';
+            const trace = await finishTrace('failed', {
+                error
+            });
+            debugInteract('recorder aiDescribe trace:', trace);
+            return {
+                event: {
+                    ...event,
+                    semantic: buildFailedAiDescribeRecorderSemantic(error)
+                },
+                trace
+            };
+        }
+        const x = event.elementRect?.x;
+        const y = event.elementRect?.y;
+        let modelCallDurationMs;
+        let modelCallStartedAt;
+        let elementDescription;
+        let deepLocate;
+        let verifyResult;
+        try {
+            if ('number' != typeof x || 'number' != typeof y) throw new Error('Skipped aiDescribe because the recorder event has no stable point target.');
+            if (!eventScreenshot) throw new Error('Skipped aiDescribe because the recorder event has no screenshot.');
+            if (!event.pageInfo?.width || !event.pageInfo?.height) throw new Error('Skipped aiDescribe because the recorder event has no pageInfo for coordinate mapping.');
+            const verifyPrompt = shouldVerifyRecorderAiDescribeEvent(event);
+            modelCallStartedAt = Date.now();
+            const describeResult = await withTimeout(agent.describeElementAtPoint([
+                x,
+                y
+            ], {
+                verifyPrompt,
+                screenshotBase64: eventScreenshot,
+                coordinateSpace: 'logical',
+                logicalSize: event.pageInfo,
+                onProgress: (progress)=>{
+                    elementDescription = progress.prompt?.trim() || elementDescription;
+                    deepLocate = progress.deepLocate;
+                    verifyResult = verifyPrompt ? progress.verifyResult : void 0;
+                }
+            }), RECORDER_AI_DESCRIBE_AFTER_INTERACT_TIMEOUT_MS, 'Timed out while analyzing recorder event with aiDescribe.');
+            modelCallDurationMs = Date.now() - modelCallStartedAt;
+            elementDescription = describeResult.prompt?.trim();
+            deepLocate = describeResult.deepLocate;
+            verifyResult = verifyPrompt ? describeResult.verifyResult : void 0;
+            if (!elementDescription) throw new Error("aiDescribe returned an empty element description.");
+            if (verifyResult && false === verifyResult.pass) throw new Error('aiDescribe verification failed.');
+            const semanticAction = buildRecorderSemanticAction(event.actionType || event.type, event.rawPayload || {}, event.url);
+            const trace = await finishTrace('ready', {
+                modelCallDurationMs,
+                elementDescription,
+                verifyPassed: verifyResult?.pass,
+                centerDistance: verifyResult?.centerDistance,
+                verifyResult
+            });
+            debugInteract('recorder aiDescribe trace:', trace);
+            return {
+                event: {
+                    ...event,
+                    semantic: buildReadyRecorderSemantic('aiDescribe', semanticAction, elementDescription, verifyResult?.pass ? 'high' : 'medium', {
+                        aiDescribe: {
+                            verifyPrompt,
+                            verifyPassed: verifyResult?.pass,
+                            deepLocate,
+                            centerDistance: verifyResult?.centerDistance,
+                            expectedCenter: [
+                                x,
+                                y
+                            ],
+                            actualCenter: verifyResult?.center
+                        }
+                    })
+                },
+                trace
+            };
+        } catch (error) {
+            const reportedError = verifyResult?.pass === false ? new Error('aiDescribe verification failed.') : error;
+            if (void 0 === modelCallDurationMs && modelCallStartedAt) modelCallDurationMs = Date.now() - modelCallStartedAt;
+            debugInteract('canonical recorder aiDescribe failed:', reportedError);
+            const trace = await finishTrace('failed', {
+                error: reportedError instanceof Error ? reportedError.message : String(reportedError),
+                modelCallDurationMs,
+                elementDescription,
+                verifyPassed: verifyResult?.pass,
+                centerDistance: verifyResult?.centerDistance,
+                verifyResult
+            });
+            debugInteract('recorder aiDescribe trace:', trace);
+            const aiDescribeDetails = verifyResult || trace.annotatedScreenshotRef ? {
+                aiDescribe: {
+                    verifyPrompt: true,
+                    verifyPassed: verifyResult?.pass,
+                    deepLocate,
+                    centerDistance: verifyResult?.centerDistance,
+                    expectedCenter: 'number' == typeof x && 'number' == typeof y ? [
+                        x,
+                        y
+                    ] : void 0,
+                    actualCenter: verifyResult?.center,
+                    annotatedScreenshotPath: trace.annotatedScreenshotRef?.path
+                }
+            } : void 0;
+            return {
+                event: {
+                    ...event,
+                    semantic: buildFailedAiDescribeRecorderSemantic(reportedError, aiDescribeDetails)
+                },
+                trace
+            };
+        }
+    }
+    getRecorderAiDescribeScreenshot(event) {
+        if ('click' === event.type || 'input' === event.type || 'keydown' === event.type || 'drag' === event.type) return event.screenshotBefore || event.screenshotAfter;
+        return event.screenshotAfter || event.screenshotBefore;
+    }
+    queueStudioPreviewRecorderEventAppend(event, navigationEvent) {
+        const sessionId = this._recorderSessionId;
+        if (!sessionId) return;
+        this._recorderEvents.push(event);
+        if (navigationEvent) this._recorderEvents.push(navigationEvent);
+    }
+    queueStudioPreviewRecorderEvent(payload, snapshotBefore, agent) {
+        const sessionId = this._recorderSessionId;
+        if (!sessionId) return;
+        this._recorderPendingCaptures++;
+        const queuedTask = this._recorderEventQueue.catch(()=>void 0).then(async ()=>{
+            try {
+                if (!this._recorderSessionId || this._recorderSessionId !== sessionId) return;
+                await this.storeStudioPreviewRecorderEvent(payload, snapshotBefore, agent);
+            } finally{
+                this._recorderPendingCaptures = Math.max(0, this._recorderPendingCaptures - 1);
+            }
+        });
+        this._recorderEventQueue = queuedTask.catch((error)=>{
+            debugInteract('async recorder event capture failed:', error);
+        });
+    }
     buildStudioPreviewNavigationChangeEvent(payload, pageStateBefore, pageStateAfter, screenshotAfter) {
         const actionType = 'string' == typeof payload.actionType ? payload.actionType : void 0;
         if (!actionType || BROWSER_CHROME_NAVIGATION_ACTIONS.has(actionType)) return null;
@@ -647,6 +1134,9 @@ class PlaygroundServer {
         const afterUrl = pageStateAfter.url;
         if (!afterUrl || beforeUrl === afterUrl) return null;
         const timestamp = Date.now();
+        const semanticEvent = buildRecorderSemanticAction('NavigationChanged', {
+            value: afterUrl
+        }, afterUrl);
         return {
             source: 'studio-preview',
             type: 'navigation',
@@ -662,12 +1152,7 @@ class PlaygroundServer {
             value: afterUrl,
             screenshotBefore: screenshotAfter,
             screenshotAfter,
-            elementDescription: afterUrl,
-            replayInstruction: `Wait for navigation to complete at \`${afterUrl}\`.`,
-            actionSummary: `Wait for navigation to complete at ${afterUrl}`,
-            semanticConfidence: 'high',
-            descriptionLoading: false,
-            descriptionSource: 'fallback',
+            semantic: buildReadyRecorderSemantic('heuristic', semanticEvent, afterUrl, 'high'),
             timestamp,
             hashId: `studio-preview-navigation-${timestamp}-${Math.random().toString(36).slice(2, 8)}`
         };
@@ -675,6 +1160,9 @@ class PlaygroundServer {
     buildStudioPreviewInitialNavigationEvent(pageState, screenshot) {
         if (!pageState.url) return null;
         const timestamp = Date.now();
+        const semanticEvent = buildRecorderSemanticAction('InitialNavigation', {
+            value: pageState.url
+        }, pageState.url);
         return {
             source: 'studio-preview',
             type: 'navigation',
@@ -689,12 +1177,7 @@ class PlaygroundServer {
             value: pageState.url,
             screenshotBefore: screenshot,
             screenshotAfter: screenshot,
-            elementDescription: pageState.url,
-            replayInstruction: `Navigate to \`${pageState.url}\`.`,
-            actionSummary: `Navigate to ${pageState.url}`,
-            semanticConfidence: 'high',
-            descriptionLoading: false,
-            descriptionSource: 'fallback',
+            semantic: buildReadyRecorderSemantic('heuristic', semanticEvent, pageState.url, 'high'),
             timestamp,
             hashId: `studio-preview-initial-navigation-${timestamp}-${Math.random().toString(36).slice(2, 8)}`
         };
@@ -1036,7 +1519,7 @@ class PlaygroundServer {
                     error: error instanceof Error ? error.message : 'No active session'
                 });
             }
-            const { type, prompt, params, requestId, deepLocate, deepThink, screenshotIncluded, domIncluded, deviceOptions } = req.body;
+            const { type, prompt, params, requestId, deepLocate, deepThink, screenshotIncluded, domIncluded, deviceOptions, reportDisplay } = req.body;
             if (!type) return res.status(400).json({
                 error: 'type is required'
             });
@@ -1085,6 +1568,7 @@ class PlaygroundServer {
             };
             const startTime = Date.now();
             try {
+                agent.resetDump?.();
                 await this._activeConnection.executionHooks?.beforeExecute?.();
                 const actionSpace = agent.interface.actionSpace();
                 const value = {
@@ -1097,7 +1581,8 @@ class PlaygroundServer {
                     deepThink,
                     screenshotIncluded,
                     domIncluded,
-                    deviceOptions
+                    deviceOptions,
+                    reportDisplay
                 });
             } catch (error) {
                 response.error = formatErrorMessage(error);
@@ -1114,7 +1599,7 @@ class PlaygroundServer {
                 });
                 if (dumpString) {
                     const groupedDump = ReportActionDump.fromSerializedString(dumpString);
-                    response.dump = groupedDump.executions?.[0] || null;
+                    response.dump = groupedDump;
                 } else response.dump = null;
                 response.reportHTML = agent.reportHTMLString({
                     inlineScreenshots: true
@@ -1154,7 +1639,7 @@ class PlaygroundServer {
                     });
                     if (dumpString) {
                         const groupedDump = ReportActionDump.fromSerializedString(dumpString);
-                        dump = groupedDump.executions?.[0] || null;
+                        dump = groupedDump;
                     }
                     reportHTML = agent.reportHTMLString?.({
                         inlineScreenshots: true
@@ -1225,6 +1710,7 @@ class PlaygroundServer {
             res.json(await this.getRecorderCapabilities());
         });
         this._app.post('/recorder/stop', async (_req, res)=>{
+            await this.waitForRecorderIdle();
             this._recorderSessionId = null;
             this._studioPreviewRecorderLastScreenshot = void 0;
             this._studioPreviewRecorderLastPageState = void 0;
@@ -1240,6 +1726,38 @@ class PlaygroundServer {
                 nextIndex: this._recorderEvents.length
             });
         });
+        this._app.post('/recorder/describe-event', async (req, res)=>{
+            const event = req.body?.event;
+            if (!event || 'object' != typeof event) return res.status(400).json({
+                ok: false,
+                error: 'event is required'
+            });
+            try {
+                const agent = this.getActiveAgentOrThrow();
+                const { event: describedEvent, trace } = await this.enrichStudioPreviewRecorderEventWithAiDescribe(event, agent);
+                res.json({
+                    ok: true,
+                    event: describedEvent,
+                    trace
+                });
+            } catch (error) {
+                const startedAt = new Date();
+                const traceBase = createRecorderAiDescribeTraceBase(event);
+                const trace = {
+                    ...traceBase,
+                    status: 'failed',
+                    startedAt: startedAt.toISOString(),
+                    durationMs: 0,
+                    error: error instanceof Error ? error.message : String(error)
+                };
+                debugInteract('recorder aiDescribe trace:', trace);
+                res.status(500).json({
+                    ok: false,
+                    error: error instanceof Error ? error.message : String(error),
+                    trace
+                });
+            }
+        });
         this._app.get('/screenshot', async (_req, res)=>{
             try {
                 let agent = this.getActiveAgentOrThrow();
@@ -1340,7 +1858,7 @@ class PlaygroundServer {
                     recorderActive: Boolean(this._recorderSessionId),
                     hasInputPrimitives: Boolean(agent.interface.inputPrimitives)
                 });
-                const recorderSnapshotBefore = await this.captureRecorderSnapshotBeforeInteract();
+                const recorderSnapshotBefore = this.captureCachedRecorderSnapshotBeforeInteract();
                 const inputPrimitives = agent.interface.inputPrimitives;
                 if (inputPrimitives) {
                     await dispatchPointer(inputPrimitives, req.body ?? {}, ()=>agent.interface.size());
@@ -1348,12 +1866,12 @@ class PlaygroundServer {
                         payload: summarizeInteractPayload(req.body ?? {}),
                         elapsedMs: Date.now() - interactStartedAt
                     });
-                    await this.storeStudioPreviewRecorderEvent(req.body ?? {}, recorderSnapshotBefore);
+                    res.json({});
+                    this.queueStudioPreviewRecorderEvent(req.body ?? {}, recorderSnapshotBefore, agent);
                     debugInteract('manual interact completed %o', {
                         payload: summarizeInteractPayload(req.body ?? {}),
                         elapsedMs: Date.now() - interactStartedAt
                     });
-                    res.json({});
                     return;
                 }
                 if (!this.findInteractAction(agent, actionType) && !this.canRunBrowserChromeInteractAction(agent, actionType)) return res.status(404).json({
@@ -1365,12 +1883,12 @@ class PlaygroundServer {
                     payload: summarizeInteractPayload(req.body ?? {}),
                     elapsedMs: Date.now() - interactStartedAt
                 });
-                await this.storeStudioPreviewRecorderEvent(req.body ?? {}, recorderSnapshotBefore);
+                res.json({});
+                this.queueStudioPreviewRecorderEvent(req.body ?? {}, recorderSnapshotBefore, agent);
                 debugInteract('manual interact completed %o', {
                     payload: summarizeInteractPayload(req.body ?? {}),
                     elapsedMs: Date.now() - interactStartedAt
                 });
-                res.json({});
             } catch (error) {
                 if (error instanceof PointerInputError) return res.status(error.statusCode).json({
                     error: error.message
@@ -1559,6 +2077,9 @@ class PlaygroundServer {
         _define_property(this, "_baseSidecars", void 0);
         _define_property(this, "_recorderSessionId", null);
         _define_property(this, "_recorderEvents", []);
+        _define_property(this, "_recorderEventQueue", Promise.resolve());
+        _define_property(this, "_recorderPendingCaptures", 0);
+        _define_property(this, "_studioPreviewRecorderLastTargetPoint", void 0);
         _define_property(this, "_studioPreviewRecorderLastScreenshot", void 0);
         _define_property(this, "_studioPreviewRecorderLastPageState", void 0);
         _define_property(this, "_activeConnection", {