@midscene/playground 1.9.6 → 1.9.7

package/dist/types/adapters/local-execution.d.ts

@@ -1,4 +1,5 @@
 import type { ConnectivityTestResult, DeviceAction, ExecutionDump } from '@midscene/core';
+import { ReportActionDump } from '@midscene/core';
 import { type PlaygroundRuntimeInfo } from '../runtime-metadata';
 import type { AgentFactory, ExecutionOptions, FormValue, PlaygroundAgent } from '../types';
 import { BasePlaygroundAdapter } from './base';
@@ -35,7 +36,7 @@ export declare class LocalExecutionAdapter extends BasePlaygroundAdapter {
     cancelTask(_requestId: string): Promise<{
         error?: string;
         success?: boolean;
-        dump?: ExecutionDump | null;
+        dump?: ExecutionDump | ReportActionDump | null;
         reportHTML?: string | null;
     }>;
     /**
@@ -43,7 +44,7 @@ export declare class LocalExecutionAdapter extends BasePlaygroundAdapter {
      * This allows retrieving dump and report when execution is stopped
      */
     getCurrentExecutionData(): Promise<{
-        dump: ExecutionDump | null;
+        dump: ExecutionDump | ReportActionDump | null;
         reportHTML: string | null;
     }>;
     getInterfaceInfo(): Promise<{

package/dist/types/adapters/remote-execution.d.ts

@@ -1,5 +1,5 @@
 import type { ConnectivityTestResult, DeviceAction, ExecutionDump } from '@midscene/core';
-import type { PlaygroundRecorderCapabilitiesResult, PlaygroundRecorderEventsResult, PlaygroundRecorderStartResult, PlaygroundSessionSetup, PlaygroundSessionState, PlaygroundSessionTarget } from '../platform';
+import type { PlaygroundRecorderCapabilitiesResult, PlaygroundRecorderDescribeResult, PlaygroundRecorderEvent, PlaygroundRecorderEventsResult, PlaygroundRecorderStartResult, PlaygroundSessionSetup, PlaygroundSessionState, PlaygroundSessionTarget } from '../platform';
 import type { PlaygroundRuntimeInfo } from '../runtime-metadata';
 import type { ExecutionOptions, FormValue, ValidationResult } from '../types';
 import { BasePlaygroundAdapter } from './base';
@@ -53,6 +53,7 @@ export declare class RemoteExecutionAdapter extends BasePlaygroundAdapter {
         error?: string;
     }>;
     getRecorderEvents(since?: number): Promise<PlaygroundRecorderEventsResult>;
+    describeRecorderEventAtPoint(event: PlaygroundRecorderEvent): Promise<PlaygroundRecorderDescribeResult>;
     getInterfaceInfo(): Promise<{
         type: string;
         description?: string;

package/dist/types/index.browser.d.ts

@@ -11,5 +11,5 @@ export declare const playgroundForSessionManager: undefined;
 export declare const launchPreparedPlaygroundPlatform: undefined;
 export type { PlaygroundInteractPayload, PlaygroundInteractResult, PlaygroundPageRecordedEvent, } from './sdk/index';
 export type { BeforeActionHook, ExecutionOptions, FormValue, PlaygroundAgent, ValidationResult, PlaygroundConfig, ExecutionType, PlaygroundAdapter, } from './types';
-export type { PlaygroundCreatedSession, PlaygroundExecutionHooks, PlaygroundPlatformRegistration, PlaygroundPlatformDescriptor, PlaygroundPlatformSelectorConfig, PlaygroundPreviewCapability, PlaygroundPreviewDescriptor, PlaygroundPreviewKind, PlaygroundRecorderCapabilitiesResult, PlaygroundRecorderEvent, PlaygroundRecorderEventsResult, PlaygroundRecorderSourceKind, PlaygroundRecorderStartResult, PlaygroundSidecar, PreparedPlaygroundPlatform, PlaygroundSessionField, PlaygroundSessionFieldOption, PlaygroundSessionManager, PlaygroundSessionSetup, PlaygroundSessionState, PlaygroundSessionTarget, } from './platform';
+export type { PlaygroundCreatedSession, PlaygroundExecutionHooks, PlaygroundPlatformRegistration, PlaygroundPlatformDescriptor, PlaygroundPlatformSelectorConfig, PlaygroundPreviewCapability, PlaygroundPreviewDescriptor, PlaygroundPreviewKind, PlaygroundRecorderCapabilitiesResult, PlaygroundRecorderDescribeResult, PlaygroundRecorderEvent, PlaygroundRecorderEventsResult, PlaygroundRecorderSourceKind, PlaygroundRecorderStartResult, PlaygroundSidecar, PreparedPlaygroundPlatform, PlaygroundSessionField, PlaygroundSessionFieldOption, PlaygroundSessionManager, PlaygroundSessionSetup, PlaygroundSessionState, PlaygroundSessionTarget, } from './platform';
 export type { PlaygroundRuntimeInfo } from './runtime-metadata';

package/dist/types/index.d.ts

@@ -13,6 +13,6 @@ export { RemoteExecutionAdapter } from './adapters/remote-execution';
 export type { PlaygroundInteractPayload, PlaygroundInteractResult, PlaygroundPageRecordedEvent, } from './sdk/index';
 export type { BeforeActionHook, ExecutionOptions, FormValue, PlaygroundAgent, ValidationResult, PlaygroundConfig, ExecutionType, PlaygroundAdapter, ServerResponse, AgentFactory, } from './types';
 export type { LaunchPlaygroundOptions, LaunchPlaygroundResult, } from './launcher';
-export type { PlaygroundCreatedSession, PlaygroundExecutionHooks, PlaygroundPlatformRegistration, PlaygroundPlatformDescriptor, PlaygroundPlatformSelectorConfig, PlaygroundPreviewCapability, PlaygroundPreviewDescriptor, PlaygroundPreviewKind, PlaygroundRecorderCapabilitiesResult, PlaygroundRecorderEvent, PlaygroundRecorderEventsResult, PlaygroundRecorderSourceKind, PlaygroundRecorderStartResult, PlaygroundSidecar, PreparedPlaygroundPlatform, PlaygroundSessionField, PlaygroundSessionFieldOption, PlaygroundSessionManager, PlaygroundSessionSetup, PlaygroundSessionState, PlaygroundSessionTarget, } from './platform';
+export type { PlaygroundCreatedSession, PlaygroundExecutionHooks, PlaygroundPlatformRegistration, PlaygroundPlatformDescriptor, PlaygroundPlatformSelectorConfig, PlaygroundPreviewCapability, PlaygroundPreviewDescriptor, PlaygroundPreviewKind, PlaygroundRecorderCapabilitiesResult, PlaygroundRecorderDescribeResult, PlaygroundRecorderEvent, PlaygroundRecorderEventsResult, PlaygroundRecorderSourceKind, PlaygroundRecorderStartResult, PlaygroundSidecar, PreparedPlaygroundPlatform, PlaygroundSessionField, PlaygroundSessionFieldOption, PlaygroundSessionManager, PlaygroundSessionSetup, PlaygroundSessionState, PlaygroundSessionTarget, } from './platform';
 export type { PlaygroundRuntimeInfo } from './runtime-metadata';
 export type { PrepareMultiPlatformPlaygroundOptions, RegisteredPlaygroundPlatform, } from './multi-platform';

package/dist/types/mjpeg-hub.d.ts

@@ -66,6 +66,7 @@ export declare class InterfaceMjpegHub {
      * forcibly closes attached subscriber sockets.
      */
     shutdown(): void;
+    getLastFrame(): MjpegStreamFrame | undefined;
     private streamRequestInternal;
     private attachSubscriber;
     /**

package/dist/types/mjpeg-stream-handler.d.ts

@@ -38,11 +38,13 @@ export declare class MjpegStreamHandler {
     private readonly source;
     private nativeAvailable;
     private nativeFailedAt;
+    private lastPollingFrame?;
     private readonly interfaceMjpegHub;
     constructor(source: MjpegStreamSource);
     /** Drop the cached probe result — call this when the agent reconnects. */
     reset(): void;
     shutdown(): void;
+    getLastFrameBase64(): string | undefined;
     serve(req: Request, res: Response): Promise<void>;
     private probeAndProxyNative;
     private probeNativeLiveness;

package/dist/types/platform.d.ts

@@ -96,6 +96,110 @@ export interface PlaygroundRecorderEventsResult {
     events: PlaygroundRecorderEvent[];
     nextIndex: number;
 }
+export interface PlaygroundRecorderDescribeTrace {
+    traceId: string;
+    eventHashId?: string;
+    eventType?: string;
+    actionType?: string;
+    eventSummary?: {
+        hashId?: string;
+        mergedHashIds?: string[];
+        type?: string;
+        source?: string;
+        actionType?: string;
+        timestamp?: number;
+        url?: string;
+        title?: string;
+        valueLength?: number;
+        rawPayloadSummary?: Record<string, unknown>;
+        elementRect?: {
+            left?: number;
+            top?: number;
+            width?: number;
+            height?: number;
+            x?: number;
+            y?: number;
+        };
+        pageInfo?: {
+            width: number;
+            height: number;
+        };
+    };
+    status: 'ready' | 'failed' | 'skipped';
+    error?: string;
+    startedAt: string;
+    durationMs: number;
+    modelCallDurationMs?: number;
+    point?: [number, number];
+    pageInfo?: {
+        width: number;
+        height: number;
+    };
+    screenshotBytes?: number;
+    screenshotRef?: {
+        path: string;
+        sha256: string;
+        bytes: number;
+        mimeType?: string;
+    };
+    annotatedScreenshotRef?: {
+        path: string;
+        sha256: string;
+        bytes: number;
+        mimeType?: string;
+    };
+    screenshotAnnotation?: {
+        inputPoint?: {
+            logical: [number, number];
+            screenshot: [number, number];
+        };
+        sourceTargetRect?: {
+            left: number;
+            top: number;
+            width: number;
+            height: number;
+        };
+        locateRect?: {
+            left: number;
+            top: number;
+            width: number;
+            height: number;
+        };
+        centerDelta?: {
+            x: number;
+            y: number;
+            distance: number;
+        };
+        distanceOutsideRect?: {
+            x: number;
+            y: number;
+            distance: number;
+        };
+    };
+    screenshotPersistError?: string;
+    annotatedScreenshotPersistError?: string;
+    elementDescription?: string;
+    verifyPassed?: boolean;
+    centerDistance?: number;
+    verifyResult?: {
+        pass?: boolean;
+        rect?: {
+            left: number;
+            top: number;
+            width: number;
+            height: number;
+        };
+        center?: [number, number];
+        centerDistance?: number;
+        includedInRect?: boolean;
+    };
+}
+export interface PlaygroundRecorderDescribeResult {
+    ok: boolean;
+    event?: PlaygroundRecorderEvent;
+    trace?: PlaygroundRecorderDescribeTrace;
+    error?: string;
+}
 export interface PlaygroundSessionState {
     connected: boolean;
     displayName?: string;

package/dist/types/sdk/index.d.ts

@@ -1,5 +1,5 @@
 import type { ConnectivityTestResult, DeviceAction } from '@midscene/core';
-import type { PlaygroundRecorderCapabilitiesResult, PlaygroundRecorderEvent, PlaygroundRecorderEventsResult, PlaygroundRecorderStartResult, PlaygroundSessionSetup, PlaygroundSessionState, PlaygroundSessionTarget } from '../platform';
+import type { PlaygroundRecorderCapabilitiesResult, PlaygroundRecorderDescribeResult, PlaygroundRecorderEvent, PlaygroundRecorderEventsResult, PlaygroundRecorderStartResult, PlaygroundSessionSetup, PlaygroundSessionState, PlaygroundSessionTarget } from '../platform';
 import type { PlaygroundRuntimeInfo } from '../runtime-metadata';
 import type { BeforeActionHook, ExecutionOptions, FormValue, PlaygroundConfig, ValidationResult } from '../types';
 export type PlaygroundInteractPayload = {
@@ -10,7 +10,7 @@ export interface PlaygroundInteractResult {
     error?: string;
 }
 export type PlaygroundPageRecordedEvent = PlaygroundRecorderEvent;
-export type { PlaygroundRecorderCapabilitiesResult, PlaygroundRecorderEventsResult, PlaygroundRecorderStartResult, };
+export type { PlaygroundRecorderCapabilitiesResult, PlaygroundRecorderDescribeResult, PlaygroundRecorderEventsResult, PlaygroundRecorderStartResult, };
 export declare class PlaygroundSDK {
     private adapter;
     private beforeActionHook?;
@@ -50,6 +50,7 @@ export declare class PlaygroundSDK {
     getRecorderCapabilities(): Promise<PlaygroundRecorderCapabilitiesResult>;
     stopRecorderSession(): Promise<PlaygroundInteractResult>;
     getRecorderEvents(since?: number): Promise<PlaygroundRecorderEventsResult>;
+    describeRecorderEventAtPoint(event: PlaygroundRecorderEvent): Promise<PlaygroundRecorderDescribeResult>;
     getInterfaceInfo(): Promise<{
         type: string;
         description?: string;

package/dist/types/server.d.ts

@@ -49,6 +49,9 @@ declare class PlaygroundServer {
     private _baseSidecars?;
     private _recorderSessionId;
     private _recorderEvents;
+    private _recorderEventQueue;
+    private _recorderPendingCaptures;
+    private _studioPreviewRecorderLastTargetPoint;
     private _studioPreviewRecorderLastScreenshot;
     private _studioPreviewRecorderLastPageState;
     private _activeConnection;
@@ -85,6 +88,7 @@ declare class PlaygroundServer {
     private getActiveAgentOrThrow;
     private getRecorderCapabilities;
     private resetRecorderState;
+    waitForRecorderIdle(): Promise<void>;
     private canRecordStudioPreviewInteractions;
     private takeRecorderScreenshot;
     private getActivePageInfo;
@@ -92,9 +96,14 @@ declare class PlaygroundServer {
     private getActivePageTitle;
     private getActiveRecorderPageState;
     private captureRecorderSnapshotBeforeInteract;
+    private captureCachedRecorderSnapshotBeforeInteract;
     private startStudioPreviewRecorder;
     private storeStudioPreviewRecorderEvent;
     private buildStudioPreviewRecorderEvent;
+    private enrichStudioPreviewRecorderEventWithAiDescribe;
+    private getRecorderAiDescribeScreenshot;
+    private queueStudioPreviewRecorderEventAppend;
+    private queueStudioPreviewRecorderEvent;
     private buildStudioPreviewNavigationChangeEvent;
     private buildStudioPreviewInitialNavigationEvent;
     private createRecorderScreenshotWithMarker;

package/dist/types/types.d.ts

@@ -24,6 +24,10 @@ export interface DeviceOptions {
     keyboardDismissStrategy?: 'esc-first' | 'back-first';
     alwaysRefreshScreenInfo?: boolean;
 }
+export interface ExecutionReportDisplay {
+    type?: string;
+    prompt?: string;
+}
 export interface ExecutionOptions {
     deepLocate?: boolean;
     deepThink?: boolean;
@@ -33,6 +37,7 @@ export interface ExecutionOptions {
     context?: any;
     requestId?: string;
     deviceOptions?: DeviceOptions;
+    reportDisplay?: ExecutionReportDisplay;
 }
 export type BeforeActionHook = (actionType: string, value: FormValue, options: ExecutionOptions) => void | Promise<void>;
 export type ExecutionType = 'local-execution' | 'remote-execution';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/playground",
-  "version": "1.9.6",
+  "version": "1.9.7",
   "repository": {
     "type": "git",
     "url": "https://github.com/web-infra-dev/midscene.git",
@@ -32,8 +32,8 @@
     "express": "^4.21.2",
     "open": "10.1.0",
     "uuid": "11.1.0",
-    "@midscene/core": "1.9.6",
-    "@midscene/shared": "1.9.6"
+    "@midscene/core": "1.9.7",
+    "@midscene/shared": "1.9.7"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",

package/static/index.html

@@ -1 +1 @@
-<!doctype html><html><head><link rel="icon" href="/favicon.ico"><title>Midscene Playground</title><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><script defer src="/static/js/lib-react.7b1abe58.js"></script><script defer src="/static/js/596.5426be9e.js"></script><script defer src="/static/js/index.134d5099.js"></script><link href="/static/css/index.ab28104a.css" rel="stylesheet"></head><body><div id="root"></div></body></html>
+<!doctype html><html><head><link rel="icon" href="/favicon.ico"><title>Midscene Playground</title><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><script defer src="/static/js/lib-react.7b1abe58.js"></script><script defer src="/static/js/905.8d12588e.js"></script><script defer src="/static/js/index.50e06ed5.js"></script><link href="/static/css/index.1293237f.css" rel="stylesheet"></head><body><div id="root"></div></body></html>