@midscene/playground 1.10.4 → 1.10.5-beta-20260714101501.0

package/dist/es/server.mjs

@@ -1,5 +1,5 @@
 import { createHash, randomUUID } from "node:crypto";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { dirname, join, resolve as external_node_path_resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { ReportActionDump, describeElementAtPoint, runConnectivityTest } from "@midscene/core";
@@ -9,7 +9,7 @@ import { getMidsceneRunSubDir } from "@midscene/shared/common";
 import { PLAYGROUND_SERVER_PORT } from "@midscene/shared/constants";
 import { ModelConfigManager, globalModelConfigManager, overrideAIConfig } from "@midscene/shared/env";
 import { generateElementByPoint } from "@midscene/shared/extractor";
-import { annotateRects, compositePointMarkerImg, imageInfoOfBase64 } from "@midscene/shared/img";
+import { annotateRects, imageInfoOfBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { buildMidsceneRecorderActionSummary, buildMidsceneRecorderReplayInstruction } from "@midscene/shared/recorder";
 import { uuid as utils_uuid } from "@midscene/shared/utils";
@@ -31,8 +31,28 @@ function _define_property(obj, key, value) {
 }
 const defaultPort = PLAYGROUND_SERVER_PORT;
 const RECORDER_CAPTURE_AFTER_INTERACT_DELAY_MS = 250;
+const RECORDER_TYPE_ONLY_INPUT_SETTLE_DELAY_MS = 750;
+const RECORDER_NAVIGATION_OBSERVATION_INTERVAL_MS = 250;
+const RECORDER_NAVIGATION_OBSERVATION_TIMEOUT_MS = 35000;
 const RECORDER_AI_DESCRIBE_AFTER_INTERACT_TIMEOUT_MS = 30000;
+const RECORDER_AI_DESCRIBE_VERIFY_PROMPT = false;
 const RECORDER_AI_DESCRIBE_SCREENSHOT_DUMP_DIR = 'recorder-ai-describe-screenshots';
+const RECORDER_SCREENSHOT_ASSET_DIR = 'recorder-screenshots';
+const RECORDER_SCREENSHOT_ASSET_MAX_SESSION_BYTES = 134217728;
+const RECORDER_SCREENSHOT_ASSET_MAX_TOTAL_BYTES = 536870912;
+const recorderScreenshotAssetQuotaExceededSessions = new Set();
+const recorderScreenshotAssetBytesBySession = new Map();
+let recorderScreenshotAssetUsageInitialized = false;
+function shouldPersistStudioPreviewRecorderScreenshot(actionType) {
+    return [
+        'Tap',
+        'DoubleClick',
+        'LongPress',
+        'RightClick',
+        'DragAndDrop',
+        'Input'
+    ].includes(actionType);
+}
 function createElementDescriberRuntime(agent) {
     const modelConfigManager = agent.modelConfigManager;
     const missingModelRuntime = void 0;
@@ -95,8 +115,10 @@ function buildRecorderEventSummary(event) {
         pageInfo: event.pageInfo
     };
 }
-function shouldVerifyRecorderAiDescribeEvent(event) {
-    return 'scroll' !== event.type;
+function sanitizeRecorderPathSegment(value, fallback, maxLength = 64) {
+    const normalized = 'string' == typeof value && value.trim() ? value.trim() : fallback;
+    const sanitized = normalized.replace(/[^a-zA-Z0-9_-]+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '');
+    return (sanitized || fallback).slice(0, maxLength);
 }
 function formatRecorderAiDescribeScreenshotPathParts() {
     const date = new Date();
@@ -148,6 +170,141 @@ function writeRecorderAiDescribeScreenshot(imageBase64, suffix) {
         mimeType
     };
 }
+function getRecorderScreenshotAssetRoot() {
+    const root = external_node_path_resolve(getMidsceneRunSubDir('output'), RECORDER_SCREENSHOT_ASSET_DIR);
+    mkdirSync(root, {
+        recursive: true
+    });
+    return root;
+}
+function assertRecorderScreenshotAssetId(assetId) {
+    if (!/^[a-zA-Z0-9_-]+$/.test(assetId)) throw new Error('Invalid recorder screenshot asset id');
+    return assetId;
+}
+function recorderScreenshotAssetExtension(mimeType) {
+    return mimeType?.includes('jpeg') ? 'jpg' : 'png';
+}
+function recorderScreenshotAssetPath(assetId, mimeType) {
+    const root = getRecorderScreenshotAssetRoot();
+    const safeAssetId = assertRecorderScreenshotAssetId(assetId);
+    return assertPathInsideDirectory(root, join(root, `${safeAssetId}.${recorderScreenshotAssetExtension(mimeType)}`));
+}
+function getRecorderScreenshotAssetUsage(sessionId) {
+    initializeRecorderScreenshotAssetUsage();
+    const safeSessionId = sanitizeRecorderPathSegment(sessionId, 'session');
+    const sessionBytes = recorderScreenshotAssetBytesBySession.get(safeSessionId) || 0;
+    const totalBytes = Array.from(recorderScreenshotAssetBytesBySession.values()).reduce((total, bytes)=>total + bytes, 0);
+    return {
+        totalBytes,
+        sessionBytes
+    };
+}
+function initializeRecorderScreenshotAssetUsage() {
+    if (recorderScreenshotAssetUsageInitialized) return;
+    recorderScreenshotAssetUsageInitialized = true;
+    const root = getRecorderScreenshotAssetRoot();
+    try {
+        for (const entry of readdirSync(root, {
+            withFileTypes: true
+        })){
+            if (!entry.isFile()) continue;
+            const match = entry.name.match(/^(.*)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(?:png|jpg)$/i);
+            if (!match?.[1]) continue;
+            const sessionId = match[1];
+            const bytes = statSync(join(root, entry.name)).size;
+            recorderScreenshotAssetBytesBySession.set(sessionId, (recorderScreenshotAssetBytesBySession.get(sessionId) || 0) + bytes);
+        }
+    } catch (error) {
+        debugRecorderAssets('failed to initialize recorder screenshot asset usage:', error);
+    }
+}
+function persistRecorderScreenshotAsset(sessionId, imageBase64) {
+    if (!imageBase64?.startsWith('data:')) return;
+    const { base64, mimeType } = extractBase64Payload(imageBase64);
+    const bytes = Buffer.from(base64, 'base64');
+    const safeSessionId = sanitizeRecorderPathSegment(sessionId, 'session');
+    const usage = getRecorderScreenshotAssetUsage(sessionId);
+    if (usage.sessionBytes + bytes.byteLength > RECORDER_SCREENSHOT_ASSET_MAX_SESSION_BYTES || usage.totalBytes + bytes.byteLength > RECORDER_SCREENSHOT_ASSET_MAX_TOTAL_BYTES) {
+        if (!recorderScreenshotAssetQuotaExceededSessions.has(safeSessionId)) {
+            recorderScreenshotAssetQuotaExceededSessions.add(safeSessionId);
+            debugRecorderAssets('recorder screenshot asset quota reached %o', {
+                sessionId: safeSessionId,
+                sessionBytes: usage.sessionBytes,
+                totalBytes: usage.totalBytes
+            });
+        }
+        return;
+    }
+    const assetId = `${safeSessionId}-${randomUUID()}`;
+    const normalizedMimeType = mimeType || 'image/png';
+    const filePath = recorderScreenshotAssetPath(assetId, normalizedMimeType);
+    try {
+        writeFileSync(filePath, bytes);
+    } catch (error) {
+        debugRecorderAssets('failed to persist recorder screenshot asset:', error);
+        return;
+    }
+    recorderScreenshotAssetBytesBySession.set(safeSessionId, usage.sessionBytes + bytes.byteLength);
+    return {
+        id: assetId,
+        mimeType: normalizedMimeType,
+        bytes: bytes.byteLength
+    };
+}
+function readRecorderScreenshotAsset(asset) {
+    const filePath = recorderScreenshotAssetPath(asset.id, asset.mimeType);
+    if (!existsSync(filePath)) return;
+    return `data:${asset.mimeType};base64,${readFileSync(filePath).toString('base64')}`;
+}
+function findRecorderScreenshotAssetPath(assetId) {
+    const safeAssetId = assertRecorderScreenshotAssetId(assetId);
+    const root = getRecorderScreenshotAssetRoot();
+    for (const extension of [
+        'png',
+        'jpg'
+    ]){
+        const filePath = assertPathInsideDirectory(root, join(root, `${safeAssetId}.${extension}`));
+        if (existsSync(filePath)) return filePath;
+    }
+}
+function removeRecorderScreenshotAssetsForSession(sessionId) {
+    const safeSessionId = sanitizeRecorderPathSegment(sessionId, 'session');
+    const prefix = `${safeSessionId}-`;
+    const root = getRecorderScreenshotAssetRoot();
+    for (const entry of readdirSync(root, {
+        withFileTypes: true
+    }))if (entry.isFile() && entry.name.startsWith(prefix)) rmSync(join(root, entry.name), {
+        force: true
+    });
+    recorderScreenshotAssetQuotaExceededSessions.delete(safeSessionId);
+    recorderScreenshotAssetBytesBySession.delete(safeSessionId);
+}
+function pruneRecorderScreenshotAssetsForSession(sessionId, retainedAssetIds) {
+    const safeSessionId = sanitizeRecorderPathSegment(sessionId, 'session');
+    const prefix = `${safeSessionId}-`;
+    const retained = new Set(retainedAssetIds.map((assetId)=>{
+        const safeAssetId = assertRecorderScreenshotAssetId(assetId);
+        if (!safeAssetId.startsWith(prefix)) throw new Error('Recorder screenshot asset does not belong to session');
+        return safeAssetId;
+    }));
+    const root = getRecorderScreenshotAssetRoot();
+    let sessionBytes = 0;
+    for (const entry of readdirSync(root, {
+        withFileTypes: true
+    })){
+        if (!entry.isFile() || !entry.name.startsWith(prefix)) continue;
+        const assetId = entry.name.replace(/\.(?:png|jpg)$/i, '');
+        const filePath = join(root, entry.name);
+        if (!retained.has(assetId)) {
+            rmSync(filePath, {
+                force: true
+            });
+            continue;
+        }
+        sessionBytes += statSync(filePath).size;
+    }
+    recorderScreenshotAssetBytesBySession.set(safeSessionId, sessionBytes);
+}
 function calculateRecorderScreenshotAnnotation(event, imageSize, verifyResult) {
     const pageWidth = event.pageInfo?.width;
     const pageHeight = event.pageInfo?.height;
@@ -304,6 +461,9 @@ const debugMjpeg = getDebug('playground:mjpeg', {
 const debugInteract = getDebug('playground:interact', {
     console: true
 });
+const debugRecorderAssets = getDebug('playground:recorder-assets', {
+    console: true
+});
 class InteractParamsValidationError extends Error {
     constructor(message){
         super(message);
@@ -693,8 +853,10 @@ class PlaygroundServer {
         };
     }
     resetRecorderState() {
+        this.clearPendingTypeOnlyRecorderInputFlushTimer();
         this._recorderSessionId = null;
         this._recorderEvents = [];
+        this._recorderPendingTypeOnlyInput = null;
         this._recorderEventQueue = Promise.resolve();
         this._studioPreviewRecorderLastTargetPoint = void 0;
         this._studioPreviewRecorderLastScreenshot = void 0;
@@ -801,8 +963,13 @@ class PlaygroundServer {
         const initialNavigationEvent = this.buildStudioPreviewInitialNavigationEvent(initialPageState, this._studioPreviewRecorderLastScreenshot);
         if (initialNavigationEvent) this._recorderEvents.push(initialNavigationEvent);
     }
+    persistStudioPreviewRecorderScreenshot(screenshot) {
+        if (!this._recorderSessionId) return;
+        return persistRecorderScreenshotAsset(this._recorderSessionId, screenshot);
+    }
     async storeStudioPreviewRecorderEvent(payload, snapshotBefore, agent) {
         if (!this._recorderSessionId) return;
+        const sessionId = this._recorderSessionId;
         const before = snapshotBefore || await this.captureRecorderSnapshotBeforeInteract();
         const screenshotBefore = before?.screenshot;
         debugInteract('recorder capture scheduled after action %o', {
@@ -830,6 +997,48 @@ class PlaygroundServer {
         this._studioPreviewRecorderLastScreenshot = screenshotAfter;
         this._studioPreviewRecorderLastPageState = pageStateAfter;
         this.queueStudioPreviewRecorderEventAppend(event, navigationEvent);
+        if (!navigationEvent) this.observeStudioPreviewNavigationChange(payload, before?.pageState, event.hashId, sessionId);
+    }
+    observeStudioPreviewNavigationChange(payload, pageStateBefore, triggerEventHashId, sessionId) {
+        const actionType = 'string' == typeof payload.actionType ? payload.actionType : void 0;
+        const beforeUrl = pageStateBefore?.url;
+        if (!actionType || !beforeUrl || BROWSER_CHROME_NAVIGATION_ACTIONS.has(actionType)) return;
+        (async ()=>{
+            const deadline = Date.now() + RECORDER_NAVIGATION_OBSERVATION_TIMEOUT_MS;
+            while(this._recorderSessionId === sessionId && Date.now() < deadline){
+                await sleep(RECORDER_NAVIGATION_OBSERVATION_INTERVAL_MS);
+                if (this._recorderSessionId !== sessionId) return;
+                const currentUrl = await this.getActivePageUrl();
+                if (!currentUrl || currentUrl === beforeUrl) continue;
+                const screenshot = await this.takeRecorderScreenshot();
+                const pageStateAfter = await this.getActiveRecorderPageState();
+                const navigationEvent = this.buildStudioPreviewNavigationChangeEvent(payload, pageStateBefore, pageStateAfter, screenshot);
+                if (!navigationEvent) return;
+                navigationEvent.rawPayload = {
+                    ...navigationEvent.rawPayload,
+                    triggerEventHashId
+                };
+                if (this.appendStudioPreviewNavigationEventAfterTrigger(triggerEventHashId, navigationEvent)) {
+                    this._studioPreviewRecorderLastScreenshot = screenshot;
+                    this._studioPreviewRecorderLastPageState = pageStateAfter;
+                }
+                return;
+            }
+        })().catch((error)=>{
+            debugInteract('delayed recorder navigation observation failed:', error);
+        });
+    }
+    appendStudioPreviewNavigationEventAfterTrigger(triggerEventHashId, navigationEvent) {
+        const triggerIndex = this._recorderEvents.findIndex((event)=>event.hashId === triggerEventHashId);
+        if (triggerIndex < 0) return false;
+        const alreadyRecorded = this._recorderEvents.some((event)=>{
+            if ('NavigationChanged' !== event.actionType) return false;
+            const rawPayload = event.rawPayload;
+            return rawPayload?.triggerEventHashId === triggerEventHashId;
+        });
+        if (alreadyRecorded) return false;
+        this._recorderEvents.splice(triggerIndex + 1, 0, navigationEvent);
+        return true;
     }
     async buildStudioPreviewRecorderEvent(payload, pageState, screenshotBefore, screenshotAfter) {
         const actionType = 'string' == typeof payload.actionType ? payload.actionType : void 0;
@@ -849,10 +1058,12 @@ class PlaygroundServer {
         const endX = 'number' == typeof payload.endX ? payload.endX : void 0;
         const endY = 'number' == typeof payload.endY ? payload.endY : void 0;
         const dragDescription = void 0 !== x && void 0 !== y && void 0 !== endX && void 0 !== endY ? `${Math.round(x)},${Math.round(y)} -> ${Math.round(endX)},${Math.round(endY)}` : void 0;
-        const screenshotWithMarker = void 0 !== x && void 0 !== y ? await this.createRecorderScreenshotWithMarker(screenshotBefore || screenshotAfter, pageInfo, {
-            x,
-            y
-        }) : void 0;
+        const shouldPersistScreenshot = shouldPersistStudioPreviewRecorderScreenshot(actionType);
+        const retainedScreenshot = shouldPersistScreenshot ? screenshotBefore || screenshotAfter : void 0;
+        const deferScreenshotPersistence = 'Input' === actionType && 'typeOnly' === payload.mode;
+        const screenshotAsset = deferScreenshotPersistence ? void 0 : this.persistStudioPreviewRecorderScreenshot(retainedScreenshot);
+        const shouldDiscardDataUrlScreenshot = Boolean(retainedScreenshot?.startsWith('data:'));
+        const hasRetainedScreenshot = Boolean(screenshotAsset || deferScreenshotPersistence && retainedScreenshot || !shouldDiscardDataUrlScreenshot && retainedScreenshot);
         const base = {
             source: 'studio-preview',
             actionType,
@@ -860,13 +1071,18 @@ class PlaygroundServer {
             pageInfo,
             url,
             title,
-            screenshotBefore,
-            screenshotAfter,
-            screenshotWithBox: screenshotWithMarker,
-            semantic: {
+            ...screenshotAsset ? {
+                screenshotAsset
+            } : deferScreenshotPersistence ? retainedScreenshot ? {
+                screenshotWithBox: retainedScreenshot
+            } : {} : shouldDiscardDataUrlScreenshot ? {} : {
+                screenshotBefore,
+                screenshotAfter
+            },
+            semantic: hasRetainedScreenshot ? {
                 source: 'aiDescribe',
                 status: 'pending'
-            },
+            } : buildFailedAiDescribeRecorderSemantic('Recorder screenshot was not retained because asset storage is unavailable or its quota was reached.'),
             timestamp,
             hashId: `studio-preview-${actionType}-${timestamp}-${Math.random().toString(36).slice(2, 8)}`
         };
@@ -967,7 +1183,7 @@ class PlaygroundServer {
     async enrichStudioPreviewRecorderEventWithAiDescribe(event, agent) {
         const startedAt = new Date();
         const startedAtMs = Date.now();
-        const eventScreenshot = this.getRecorderAiDescribeScreenshot(event);
+        const eventScreenshot = await this.getRecorderAiDescribeScreenshot(event);
         const traceBase = createRecorderAiDescribeTraceBase(event, eventScreenshot);
         const finishTrace = async (status, extra = {})=>{
             let screenshotRef;
@@ -1032,11 +1248,11 @@ class PlaygroundServer {
         let elementDescription;
         let deepLocate;
         let verifyResult;
+        const verifyPrompt = RECORDER_AI_DESCRIBE_VERIFY_PROMPT;
         try {
             if ('number' != typeof x || 'number' != typeof y) throw new Error('Skipped aiDescribe because the recorder event has no stable point target.');
             if (!eventScreenshot) throw new Error('Skipped aiDescribe because the recorder event has no screenshot.');
             if (!event.pageInfo?.width || !event.pageInfo?.height) throw new Error('Skipped aiDescribe because the recorder event has no pageInfo for coordinate mapping.');
-            const verifyPrompt = shouldVerifyRecorderAiDescribeEvent(event);
             modelCallStartedAt = Date.now();
             const elementDescriber = createElementDescriberRuntime(agent);
             const describeResult = await withTimeout(describeElementAtPoint(elementDescriber, [
@@ -1063,6 +1279,7 @@ class PlaygroundServer {
                 const trace = await finishTrace('ready', {
                     modelCallDurationMs,
                     elementDescription,
+                    verifyPrompt,
                     verifyPassed: false,
                     centerDistance: verifyResult.centerDistance,
                     verifyResult
@@ -1093,6 +1310,7 @@ class PlaygroundServer {
             const trace = await finishTrace('ready', {
                 modelCallDurationMs,
                 elementDescription,
+                verifyPrompt,
                 verifyPassed: verifyResult?.pass,
                 centerDistance: verifyResult?.centerDistance,
                 verifyResult
@@ -1126,6 +1344,7 @@ class PlaygroundServer {
                 error: reportedError instanceof Error ? reportedError.message : String(reportedError),
                 modelCallDurationMs,
                 elementDescription,
+                verifyPrompt,
                 verifyPassed: verifyResult?.pass,
                 centerDistance: verifyResult?.centerDistance,
                 verifyResult
@@ -1133,7 +1352,7 @@ class PlaygroundServer {
             debugInteract('recorder aiDescribe trace:', trace);
             const aiDescribeDetails = verifyResult || trace.annotatedScreenshotRef ? {
                 aiDescribe: {
-                    verifyPrompt: true,
+                    verifyPrompt,
                     verifyPassed: verifyResult?.pass,
                     deepLocate,
                     centerDistance: verifyResult?.centerDistance,
@@ -1154,16 +1373,92 @@ class PlaygroundServer {
             };
         }
     }
-    getRecorderAiDescribeScreenshot(event) {
+    async getRecorderAiDescribeScreenshot(event) {
+        if (event.screenshotAsset) {
+            const screenshot = readRecorderScreenshotAsset(event.screenshotAsset);
+            if (screenshot) return screenshot;
+        }
         if ('click' === event.type || 'input' === event.type || 'keydown' === event.type || 'drag' === event.type) return event.screenshotBefore || event.screenshotAfter;
-        return event.screenshotAfter || event.screenshotBefore;
+        return event.screenshotAfter || event.screenshotBefore || event.screenshotWithBox;
     }
     queueStudioPreviewRecorderEventAppend(event, navigationEvent) {
         const sessionId = this._recorderSessionId;
         if (!sessionId) return;
+        if (this.isDeferredTypeOnlyRecorderInput(event) && !navigationEvent) {
+            const pending = this._recorderPendingTypeOnlyInput;
+            if (pending && this.canCoalesceDeferredTypeOnlyRecorderInput(pending, event)) this._recorderPendingTypeOnlyInput = this.mergeDeferredTypeOnlyRecorderInput(pending, event);
+            else {
+                this.flushPendingTypeOnlyRecorderInput();
+                this._recorderPendingTypeOnlyInput = event;
+            }
+            this.schedulePendingTypeOnlyRecorderInputFlush();
+            return;
+        }
+        this.flushPendingTypeOnlyRecorderInput();
         this._recorderEvents.push(event);
         if (navigationEvent) this._recorderEvents.push(navigationEvent);
     }
+    isDeferredTypeOnlyRecorderInput(event) {
+        return 'studio-preview' === event.source && 'input' === event.type && 'Input' === event.actionType && event.rawPayload?.mode === 'typeOnly';
+    }
+    canCoalesceDeferredTypeOnlyRecorderInput(current, next) {
+        return current.url === next.url && current.title === next.title && current.rawPayload?.x === next.rawPayload?.x && current.rawPayload?.y === next.rawPayload?.y;
+    }
+    mergeDeferredTypeOnlyRecorderInput(current, next) {
+        const value = `${current.value || ''}${next.value || ''}`;
+        return {
+            ...current,
+            value,
+            rawPayload: {
+                ...current.rawPayload,
+                ...next.rawPayload,
+                value
+            },
+            pageInfo: next.pageInfo || current.pageInfo,
+            screenshotWithBox: next.screenshotWithBox || next.screenshotAfter || next.screenshotBefore || current.screenshotWithBox,
+            timestamp: next.timestamp,
+            mergedHashIds: Array.from(new Set([
+                current.hashId,
+                ...current.mergedHashIds || [],
+                next.hashId,
+                ...next.mergedHashIds || []
+            ]))
+        };
+    }
+    flushPendingTypeOnlyRecorderInput() {
+        this.clearPendingTypeOnlyRecorderInputFlushTimer();
+        const pending = this._recorderPendingTypeOnlyInput;
+        if (!pending) return;
+        this._recorderPendingTypeOnlyInput = null;
+        const screenshot = pending.screenshotWithBox || pending.screenshotAfter || pending.screenshotBefore;
+        const screenshotAsset = this.persistStudioPreviewRecorderScreenshot(screenshot);
+        if (screenshotAsset) return void this._recorderEvents.push({
+            ...pending,
+            screenshotAsset,
+            screenshotBefore: void 0,
+            screenshotAfter: void 0,
+            screenshotWithBox: void 0
+        });
+        this._recorderEvents.push({
+            ...pending,
+            screenshotBefore: void 0,
+            screenshotAfter: void 0,
+            screenshotWithBox: void 0,
+            semantic: buildFailedAiDescribeRecorderSemantic('Recorder screenshot was not retained because asset storage is unavailable or its quota was reached.')
+        });
+    }
+    schedulePendingTypeOnlyRecorderInputFlush() {
+        this.clearPendingTypeOnlyRecorderInputFlushTimer();
+        this._recorderPendingTypeOnlyInputFlushTimer = setTimeout(()=>{
+            this._recorderPendingTypeOnlyInputFlushTimer = void 0;
+            this.flushPendingTypeOnlyRecorderInput();
+        }, RECORDER_TYPE_ONLY_INPUT_SETTLE_DELAY_MS);
+    }
+    clearPendingTypeOnlyRecorderInputFlushTimer() {
+        if (!this._recorderPendingTypeOnlyInputFlushTimer) return;
+        clearTimeout(this._recorderPendingTypeOnlyInputFlushTimer);
+        this._recorderPendingTypeOnlyInputFlushTimer = void 0;
+    }
     queueStudioPreviewRecorderEvent(payload, snapshotBefore, agent) {
         const sessionId = this._recorderSessionId;
         if (!sessionId) return;
@@ -1203,8 +1498,6 @@ class PlaygroundServer {
             url: afterUrl,
             title: pageStateAfter.title,
             value: afterUrl,
-            screenshotBefore: screenshotAfter,
-            screenshotAfter,
             semantic: buildReadyRecorderSemantic('heuristic', semanticEvent, afterUrl, 'high'),
             timestamp,
             hashId: `studio-preview-navigation-${timestamp}-${Math.random().toString(36).slice(2, 8)}`
@@ -1228,26 +1521,11 @@ class PlaygroundServer {
             url: pageState.url,
             title: pageState.title,
             value: pageState.url,
-            screenshotBefore: screenshot,
-            screenshotAfter: screenshot,
             semantic: buildReadyRecorderSemantic('heuristic', semanticEvent, pageState.url, 'high'),
             timestamp,
             hashId: `studio-preview-initial-navigation-${timestamp}-${Math.random().toString(36).slice(2, 8)}`
         };
     }
-    async createRecorderScreenshotWithMarker(screenshot, pageInfo, point) {
-        if (!screenshot || !pageInfo.width || !pageInfo.height) return;
-        try {
-            return await compositePointMarkerImg({
-                inputImgBase64: screenshot,
-                size: pageInfo,
-                point
-            });
-        } catch (error) {
-            debugScreenshot('recorder screenshot marker failed:', error);
-            return;
-        }
-    }
     async destroyCurrentAgent({ preserveActiveStream = false } = {}) {
         if (!this._activeConnection.agent) return;
         try {
@@ -1605,9 +1883,12 @@ class PlaygroundServer {
                 error: 'Another task is already running',
                 currentTaskId: this.currentTaskId
             });
+            let abortController = null;
             if (requestId) {
                 this.currentTaskId = requestId;
                 this.taskExecutionDumps[requestId] = null;
+                abortController = new AbortController();
+                this.taskAbortControllers[requestId] = abortController;
                 agent.onDumpUpdate = (_dump, executionDump)=>{
                     if (executionDump) this.taskExecutionDumps[requestId] = executionDump;
                 };
@@ -1630,10 +1911,12 @@ class PlaygroundServer {
                     params
                 };
                 response.result = await executeAction(agent, type, actionSpace, value, {
+                    requestId,
                     deepLocate,
                     deepThink,
                     screenshotIncluded,
                     domIncluded,
+                    abortSignal: abortController?.signal,
                     deviceOptions,
                     reportDisplay
                 });
@@ -1669,6 +1952,7 @@ class PlaygroundServer {
             else console.log(`handle request done after ${timeCost}ms: requestId: ${requestId}`);
             if (requestId) {
                 delete this.taskExecutionDumps[requestId];
+                delete this.taskAbortControllers[requestId];
                 if (this.currentTaskId === requestId) this.currentTaskId = null;
             }
         });
@@ -1700,6 +1984,8 @@ class PlaygroundServer {
                 } catch (error) {
                     console.warn('Failed to get execution data before cancel:', error);
                 }
+                const abortController = this.taskAbortControllers[requestId];
+                abortController?.abort(new Error(`Task cancelled by user: ${requestId}`));
                 try {
                     await this.recreateAgent({
                         preserveActiveStream: true
@@ -1708,6 +1994,7 @@ class PlaygroundServer {
                     console.warn('Failed to recreate agent during cancel:', error);
                 }
                 delete this.taskExecutionDumps[requestId];
+                delete this.taskAbortControllers[requestId];
                 this.currentTaskId = null;
                 res.json({
                     status: 'cancelled',
@@ -1764,6 +2051,7 @@ class PlaygroundServer {
         });
         this._app.post('/recorder/stop', async (_req, res)=>{
             await this.waitForRecorderIdle();
+            this.flushPendingTypeOnlyRecorderInput();
             this._recorderSessionId = null;
             this._studioPreviewRecorderLastScreenshot = void 0;
             this._studioPreviewRecorderLastPageState = void 0;
@@ -1772,6 +2060,7 @@ class PlaygroundServer {
             });
         });
         this._app.get('/recorder/events', async (req, res)=>{
+            if ('false' !== req.query.flushPending) this.flushPendingTypeOnlyRecorderInput();
             const since = 'string' == typeof req.query.since ? Number.parseInt(req.query.since, 10) : 0;
             const startIndex = Number.isFinite(since) && since > 0 ? since : 0;
             res.json({
@@ -1779,6 +2068,48 @@ class PlaygroundServer {
                 nextIndex: this._recorderEvents.length
             });
         });
+        this._app.get('/recorder/assets/:assetId', async (req, res)=>{
+            try {
+                const assetId = String(req.params.assetId || '');
+                const filePath = findRecorderScreenshotAssetPath(assetId);
+                if (!filePath) return res.status(404).json({
+                    error: 'Recorder screenshot not found'
+                });
+                res.type(filePath.endsWith('.jpg') ? 'image/jpeg' : 'image/png');
+                return res.send(readFileSync(filePath));
+            } catch (error) {
+                return res.status(400).json({
+                    error: error instanceof Error ? error.message : 'Invalid recorder screenshot request'
+                });
+            }
+        });
+        this._app.delete('/recorder/assets/session/:sessionId', async (req, res)=>{
+            try {
+                removeRecorderScreenshotAssetsForSession(String(req.params.sessionId || ''));
+                return res.json({
+                    ok: true
+                });
+            } catch (error) {
+                return res.status(400).json({
+                    ok: false,
+                    error: error instanceof Error ? error.message : 'Invalid recorder session id'
+                });
+            }
+        });
+        this._app.post('/recorder/assets/session/:sessionId/prune', async (req, res)=>{
+            try {
+                const assetIds = Array.isArray(req.body?.assetIds) ? req.body.assetIds.filter((assetId)=>'string' == typeof assetId) : [];
+                pruneRecorderScreenshotAssetsForSession(String(req.params.sessionId || ''), assetIds);
+                return res.json({
+                    ok: true
+                });
+            } catch (error) {
+                return res.status(400).json({
+                    ok: false,
+                    error: error instanceof Error ? error.message : 'Invalid recorder screenshot cleanup request'
+                });
+            }
+        });
         this._app.post('/recorder/describe-event', async (req, res)=>{
             const event = req.body?.event;
             if (!event || 'object' != typeof event) return res.status(400).json({
@@ -2126,6 +2457,7 @@ class PlaygroundServer {
         _define_property(this, "sessionSetupState", 'ready');
         _define_property(this, "sessionSetupBlockingReason", void 0);
         _define_property(this, "currentTaskId", null);
+        _define_property(this, "taskAbortControllers", {});
         _define_property(this, "_agentReady", true);
         _define_property(this, "_configDirty", false);
         _define_property(this, "_lastAiConfigSignature", null);
@@ -2135,6 +2467,8 @@ class PlaygroundServer {
         _define_property(this, "_baseSidecars", void 0);
         _define_property(this, "_recorderSessionId", null);
         _define_property(this, "_recorderEvents", []);
+        _define_property(this, "_recorderPendingTypeOnlyInput", null);
+        _define_property(this, "_recorderPendingTypeOnlyInputFlushTimer", void 0);
         _define_property(this, "_recorderEventQueue", Promise.resolve());
         _define_property(this, "_recorderPendingCaptures", 0);
         _define_property(this, "_studioPreviewRecorderLastTargetPoint", void 0);