@midscene/playground 1.9.8 → 1.10.0

package/dist/types/index.d.ts

@@ -1,5 +1,6 @@
 export { dataExtractionAPIs, noReplayAPIs, validationAPIs, formatErrorMessage, validateStructuredParams, executeAction, } from './common';
-export { PlaygroundServer } from './server';
+export { buildPlaygroundBrowserUrl, PlaygroundServer } from './server';
+export { describeRecorderUIEvent, describeRecorderUIEvents, getRecorderUIEventTargetRect, type DescribeRecorderUIEventInput, type DescribeRecorderUIEventOptions, type DescribeRecorderUIEventResult, } from './recorder-ui-describer';
 export { playgroundForAgent } from './launcher';
 export { playgroundForAgentFactory } from './launcher';
 export { playgroundForSessionManager } from './launcher';

package/dist/types/recorder-ai-service.d.ts

@@ -0,0 +1 @@
+export { callAIWithObjectResponse } from '@midscene/core/ai-model';

package/dist/types/recorder-ui-describer-prompt.d.ts

@@ -0,0 +1 @@
+export declare const RECORDER_UI_DESCRIBER_SYSTEM_PROMPT = "You convert Studio preview recorder UI events into semantic replay instructions.\n\nThe recorder works from screenshots and mapped real-device coordinates only. Infer stable UI intent from the highlighted BEFORE screenshot. The AFTER screenshot is contextual evidence for state changes and scroll destinations.\n\nOutput JSON only:\n{\n  \"elementDescription\": \"short stable target/region description\",\n  \"replayInstruction\": \"one executable natural-language replay step\",\n  \"actionSummary\": \"short timeline summary\",\n  \"scrollDestinationDescription\": \"for scroll only: concrete newly visible destination content or goal\",\n  \"confidence\": \"high\" | \"medium\" | \"low\",\n  \"error\"?: \"only if no useful visual description can be inferred\"\n}\n\nRules:\n- Do NOT output coordinates as the main description.\n- Do NOT mention \"near coordinates\", \"nearby element\", \"near point\", \"red marker\", highlighted box, highlighted element, or screenshot.\n- Prefer stable target descriptions in this order: exact stable control text > stable label > role + stable section/context > icon purpose > visual position.\n- Treat placeholder or hint text that can change by user, time, data, or context as dynamic. Do not use dynamic hint values as the primary target description; prefer role + stable region + intent.\n- For repeated collections, treat item identity text as dynamic unless the user is clearly verifying that exact item. This includes any list/grid/table/feed/menu or repeated record surface. Do not output descriptions like \"<role> titled/named '<content>'\"; prefer stable role + region + selection intent.\n- Keep quoted UI text in the original UI language when it is a stable control label.\n- Apply the platform guidance from the user event:\n  - Web: button, input, link, menu item, tab, dialog, aria-label, placeholder, form section.\n  - Mobile: tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, screen section.\n  - Desktop/computer: menu item, toolbar button, dialog field, sidebar item, window control, file row, application region.\n- Pointer action rules:\n  - Preserve event.actionType semantics. Tap, DoubleClick, LongPress, and RightClick must not all become Click.\n  - Tap replayInstruction format: Tap on the element described as \"<elementDescription>\".\n  - DoubleClick replayInstruction format: Double click on the element described as \"<elementDescription>\".\n  - LongPress replayInstruction format: Long press the element described as \"<elementDescription>\".\n  - RightClick replayInstruction format: Right click on the element described as \"<elementDescription>\".\n  - Click replayInstruction format: Click on the element described as \"<elementDescription>\".\n- Input-specific rules:\n  - The highlighted BEFORE screenshot marks the field that receives the text.\n  - The screenshot after the action may show the typed value; use it only to confirm the field, never as the field description.\n  - elementDescription must identify the field itself by stable field role, nearby label, region, section, or sequence intent.\n  - Never use \"AI is analyzing element\", the typed value, the page title alone, or a generic \"input field\" as elementDescription.\n  - For consecutive input events, distinguish fields by stable role, section, order, current focus, filled/empty state, and neighboring actions instead of reusing the same generic field description.\n  - Input replayInstruction format: Input \"<value>\" into the element described as \"<elementDescription>\".\n- Scroll target quality bar:\n  - elementDescription describes the scrollable page, panel, list, table, or section at the highlighted scroll point.\n  - When multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu. Do not generalize a panel/list scroll into the whole page.\n  - scrollDestinationDescription is required and describes what the scroll is trying to reveal or reach, using newly visible headings, section titles, list items, or stable content from the AFTER screenshot.\n  - Prefer descriptions like \"Playwright integration documentation page, scrolling toward the API reference section\" or \"Android API documentation page, scrolling to the installation steps section\".\n  - Do NOT write generic phrases like \"more content\", \"the page\", \"current screen\", or \"main scrollable area\".\n- Scroll replayInstruction format: Scroll the page/region with description \"<elementDescription>\" by value \"<recorded value>\" until \"<scrollDestinationDescription>\" is visible.\n- Scroll actionSummary format: Scroll <elementDescription> toward <scrollDestinationDescription>.\n- Drag/Swipe rules:\n  - Drag replayInstruction format: Drag through the area described as \"<elementDescription>\".\n  - Swipe replayInstruction format: Swipe through the area described as \"<elementDescription>\".\n  - Describe start/end regions or the dragged UI control; do not describe only the gesture path.\n- KeyboardPress replayInstruction format: Press \"<value>\" on the element described as \"<elementDescription>\".\n- If uncertain, provide the best concrete visible text/role/context description. Set confidence to \"low\"; do not fall back to coordinates.";

package/dist/types/recorder-ui-describer.d.ts

@@ -0,0 +1,20 @@
+import type { Rect } from '@midscene/core';
+import { type IModelConfig } from '@midscene/shared/env';
+import type { MidsceneRecorderEvent, MidsceneRecorderTarget } from '@midscene/shared/recorder';
+export interface DescribeRecorderUIEventInput {
+    event: MidsceneRecorderEvent;
+    target?: MidsceneRecorderTarget;
+}
+export interface DescribeRecorderUIEventOptions {
+    maxRetries?: number;
+    retryDelayMs?: number;
+    concurrency?: number;
+}
+export interface DescribeRecorderUIEventResult {
+    event: MidsceneRecorderEvent;
+    usedFallback: boolean;
+    error?: string;
+}
+export declare function getRecorderUIEventTargetRect(event: MidsceneRecorderEvent): Rect | null;
+export declare function describeRecorderUIEvent(input: DescribeRecorderUIEventInput, modelConfig: IModelConfig | Record<string, unknown>, options?: DescribeRecorderUIEventOptions): Promise<DescribeRecorderUIEventResult>;
+export declare function describeRecorderUIEvents(inputs: DescribeRecorderUIEventInput[], modelConfig: IModelConfig | Record<string, unknown>, options?: DescribeRecorderUIEventOptions): Promise<DescribeRecorderUIEventResult[]>;

package/dist/types/server.d.ts

@@ -5,6 +5,10 @@ import express from 'express';
 import type { PlaygroundPreviewDescriptor, PlaygroundSessionState, PreparedPlaygroundPlatform } from './platform';
 import { type PlaygroundRuntimeInfo } from './runtime-metadata';
 import 'dotenv/config';
+/** Default `0.0.0.0`. Override via `MIDSCENE_PLAYGROUND_HOST` (e.g. `127.0.0.1`). */
+export declare function resolvePlaygroundListenHost(): string;
+export declare function resolvePlaygroundBrowserHost(): string;
+export declare function buildPlaygroundBrowserUrl(host: string, port: number): string;
 /**
  * Recursively serialize a Zod field into a plain object that preserves
  * the `_def` metadata the client relies on (typeName, innerType, values,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/playground",
-  "version": "1.9.8",
+  "version": "1.10.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/web-infra-dev/midscene.git",
@@ -20,6 +20,12 @@
       "require": "./dist/lib/index.js",
       "import": "./dist/es/index.mjs",
       "default": "./dist/es/index.mjs"
+    },
+    "./recorder-ui-describer": {
+      "types": "./dist/types/recorder-ui-describer.d.ts",
+      "require": "./dist/lib/recorder-ui-describer.js",
+      "import": "./dist/es/recorder-ui-describer.mjs",
+      "default": "./dist/es/recorder-ui-describer.mjs"
     }
   },
   "files": [
@@ -32,8 +38,8 @@
     "express": "^4.21.2",
     "open": "10.1.0",
     "uuid": "11.1.0",
-    "@midscene/core": "1.9.8",
-    "@midscene/shared": "1.9.8"
+    "@midscene/core": "1.10.0",
+    "@midscene/shared": "1.10.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",

package/static/index.html

@@ -1 +1 @@
-<!doctype html><html><head><link rel="icon" href="/favicon.ico"><title>Midscene Playground</title><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><script defer src="/static/js/lib-react.7b1abe58.js"></script><script defer src="/static/js/905.8d12588e.js"></script><script defer src="/static/js/index.44b83907.js"></script><link href="/static/css/index.1293237f.css" rel="stylesheet"></head><body><div id="root"></div></body></html>
+<!doctype html><html><head><link rel="icon" href="/favicon.ico"><title>Midscene Playground</title><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><script defer src="/static/js/lib-react.7b1abe58.js"></script><script defer src="/static/js/905.8d12588e.js"></script><script defer src="/static/js/index.9c67d55f.js"></script><link href="/static/css/index.1293237f.css" rel="stylesheet"></head><body><div id="root"></div></body></html>