@midscene/core 1.9.6 → 1.9.7

package/dist/types/agent/agent.d.ts

@@ -1,11 +1,26 @@
 import type { TUserPrompt } from '../ai-model/index';
 import Service from '../service/index';
-import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type UIContext } from '../types';
+import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type Size, type UIContext } from '../types';
 import type { AbstractInterface } from '../device';
 import type { TaskRunner } from '../task-runner';
 import { ModelConfigManager } from '@midscene/shared/env';
 import { TaskCache } from './task-cache';
 import { TaskExecutor } from './tasks';
+export type DescribeElementCoordinateSpace = 'screenshot' | 'logical';
+export type DescribeElementAtPointOptions = {
+    verifyPrompt?: boolean;
+    retryLimit?: number;
+    deepLocate?: boolean;
+    screenshotBase64?: string;
+    screenshotSize?: Size;
+    coordinateSpace?: DescribeElementCoordinateSpace;
+    logicalSize?: Size;
+    onProgress?: (progress: {
+        prompt?: string;
+        deepLocate?: boolean;
+        verifyResult?: LocateValidatorResult;
+    }) => void;
+} & LocatorValidatorOption;
 export type AiActOptions = {
     cacheable?: boolean;
     fileChooserAccept?: string | string[];
@@ -142,11 +157,7 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     aiNumber(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<number>;
     aiString(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<string>;
     aiAsk(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<string>;
-    describeElementAtPoint(center: [number, number], opt?: {
-        verifyPrompt?: boolean;
-        retryLimit?: number;
-        deepLocate?: boolean;
-    } & LocatorValidatorOption): Promise<AgentDescribeElementAtPointResult>;
+    describeElementAtPoint(center: [number, number], opt?: DescribeElementAtPointOptions): Promise<AgentDescribeElementAtPointResult>;
     verifyLocator(prompt: string, locateOpt: LocateOption | undefined, expectCenter: [number, number], verifyLocateOption?: LocatorValidatorOption): Promise<LocateValidatorResult>;
     /**
      * Locate an element and return both its center point and an approximate rect.

package/dist/types/agent/index.d.ts

@@ -6,4 +6,4 @@ export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
 export { cacheFileExt } from './task-cache';
 export { TaskExecutor } from './tasks';
 export type { AgentOpt } from '../types';
-export type { AiActOptions } from './agent';
+export type { AiActOptions, DescribeElementAtPointOptions, DescribeElementCoordinateSpace, } from './agent';

package/dist/types/agent/tasks.d.ts

@@ -4,7 +4,7 @@ import type { AbstractInterface } from '../device';
 import type Service from '../service';
 import type { TaskRunner } from '../task-runner';
 import { TaskExecutionError } from '../task-runner';
-import type { DeviceAction, ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam } from '../types';
+import type { DeviceAction, ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam, UIContext } from '../types';
 import type { TaskCache } from './task-cache';
 export { locatePlanForLocate } from './task-builder';
 import { type TaskTitleType } from './ui-utils';
@@ -75,7 +75,9 @@ export declare class TaskExecutor {
     loadYamlFlowAsPlanning(userInstruction: TUserPrompt, yamlString: string, reportOptions?: ActionReportOptions): Promise<{
         runner: TaskRunner;
     }>;
-    runPlans(title: string, plans: PlanningAction[], planningModel: ModelRuntime, defaultModel: ModelRuntime): Promise<ExecutionResult>;
+    runPlans(title: string, plans: PlanningAction[], planningModel: ModelRuntime, defaultModel: ModelRuntime, options?: {
+        uiContext?: UIContext;
+    }): Promise<ExecutionResult>;
     action(userPrompt: TUserPrompt, planningModel: ModelRuntime, defaultModel: ModelRuntime, includeLocateInPlanning: boolean, aiActContext?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, imagesIncludeCount?: number, deepThink?: boolean, fileChooserAccept?: string[], deepLocate?: boolean, abortSignal?: AbortSignal, reportOptions?: ActionReportOptions): Promise<ExecutionResult<{
         yamlFlow?: MidsceneYamlFlowItem[];
         output?: string;

package/dist/types/agent/utils.d.ts

@@ -1,11 +1,14 @@
 import type { TMultimodalPrompt, TUserPrompt } from '../common';
 import type { AbstractInterface } from '../device';
-import type { ElementCacheFeature, LocateResultElement, PixelBbox, PlanningLocateParam, PlanningLocateParamWithLocatedPixelBbox, Rect, UIContext } from '../types';
+import type { ElementCacheFeature, LocateResultElement, PixelBbox, PlanningLocateParam, PlanningLocateParamWithLocatedPixelBbox, Rect, Size, UIContext } from '../types';
 import type { TaskCache } from './task-cache';
 export declare function commonContextParser(interfaceInstance: AbstractInterface, _opt: {
     uploadServerUrl?: string;
     screenshotShrinkFactor?: number;
 }): Promise<UIContext>;
+export declare function createScreenshotBoundUIContext(screenshotBase64: string, opt: {
+    screenshotSize?: Size;
+}): Promise<UIContext>;
 export declare function getReportFileName(tag?: string): string;
 export declare function printReportMsg(filepath: string): void;
 export declare function isPixelBbox(value: unknown): value is PixelBbox;

package/dist/types/ai-model/prompt/recorder-generation-common.d.ts

@@ -1,4 +1,4 @@
-import { type MidsceneRecorderEvent, type MidsceneRecorderMarkdownScreenshotAsset, type MidsceneRecorderTarget } from '@midscene/shared/recorder';
+import { type MidsceneRecorderEvent, type MidsceneRecorderMarkdownScreenshotAsset, type MidsceneRecorderSemantic, type MidsceneRecorderTarget } from '@midscene/shared/recorder';
 export interface EventCounts {
     navigation: number;
     click: number;
@@ -16,16 +16,19 @@ export interface ProcessedEvent {
     timestamp: number;
     source?: string;
     actionType?: string;
-    descriptionSource?: string;
-    descriptionError?: string;
     url?: string;
     title?: string;
-    elementDescription?: string;
-    replayInstruction?: string;
-    actionSummary?: string;
-    semanticConfidence?: string;
+    semantic?: MidsceneRecorderSemantic;
     description?: string;
     value?: string;
+    typedText?: string;
+    inputIndex?: number;
+    isSequentialInput?: boolean;
+    hasNeighborInput?: boolean;
+    previousInputDescription?: string;
+    previousActionDescription?: string;
+    nextActionDescription?: string;
+    neighborInputValues?: string[];
     pageInfo?: any;
     elementRect?: any;
     screenshotPath?: string;
@@ -70,6 +73,7 @@ export interface FilteredEvents {
     inputEvents: ChromeRecordedEvent[];
     scrollEvents: ChromeRecordedEvent[];
 }
+export declare function compactRecorderSemanticForGeneration(semantic?: MidsceneRecorderSemantic): MidsceneRecorderSemantic | undefined;
 export declare const validateEvents: (events: ChromeRecordedEvent[]) => void;
 export declare const getScreenshotsForLLM: (events: ChromeRecordedEvent[], maxScreenshots?: number) => string[];
 export declare const filterEventsByType: (events: ChromeRecordedEvent[]) => FilteredEvents;

package/dist/types/ai-model/prompt/recorder-ui-describer.d.ts

@@ -1 +1 @@
-export declare const RECORDER_UI_DESCRIBER_SYSTEM_PROMPT = "You convert Studio preview recorder UI events into semantic replay instructions.\n\nThe recorder works from screenshots and mapped real-device coordinates only. Infer stable UI intent from the highlighted BEFORE screenshot. The AFTER screenshot is contextual evidence for state changes and scroll destinations.\n\nOutput JSON only:\n{\n  \"elementDescription\": \"short stable target/region description\",\n  \"replayInstruction\": \"one executable natural-language replay step\",\n  \"actionSummary\": \"short timeline summary\",\n  \"scrollDestinationDescription\": \"for scroll only: concrete newly visible destination content or goal\",\n  \"confidence\": \"high\" | \"medium\" | \"low\",\n  \"error\"?: \"only if no useful visual description can be inferred\"\n}\n\nRules:\n- Do NOT output coordinates as the main description.\n- Do NOT mention \"near coordinates\", \"nearby element\", \"near point\", \"red marker\", highlighted box, highlighted element, or screenshot.\n- Prefer stable target descriptions in this order: exact visible text > label/placeholder > role + stable section/context > icon purpose > visual position.\n- Keep quoted UI text in the original UI language, for example \"\u4F7F\u7528\u6587\u6863\" or \"\u5F00\u59CB\u4F7F\u7528\".\n- Apply the platform guidance from the user event:\n  - Web: button, input, link, menu item, tab, dialog, aria-label, placeholder, form section.\n  - Mobile: tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, screen section.\n  - Desktop/computer: menu item, toolbar button, dialog field, sidebar item, window control, file row, application region.\n- Pointer action rules:\n  - Preserve event.actionType semantics. Tap, DoubleClick, LongPress, and RightClick must not all become Click.\n  - Tap replayInstruction format: Tap on the element described as \"<elementDescription>\".\n  - DoubleClick replayInstruction format: Double click on the element described as \"<elementDescription>\".\n  - LongPress replayInstruction format: Long press the element described as \"<elementDescription>\".\n  - RightClick replayInstruction format: Right click on the element described as \"<elementDescription>\".\n  - Click replayInstruction format: Click on the element described as \"<elementDescription>\".\n- Input-specific rules:\n  - The highlighted BEFORE screenshot marks the field that receives the text.\n  - The screenshot after the action may show the typed value; use it only to confirm the field, never as the field description.\n  - elementDescription must identify the field itself, for example \"\u5E74\u9F84 input in the basic form\" or \"search input in the top navigation\".\n  - Never use \"AI is analyzing element\", the typed value, or a generic \"input field\" as elementDescription.\n  - Input replayInstruction format: Input \"<value>\" into the element described as \"<elementDescription>\".\n- Scroll target quality bar:\n  - elementDescription describes the scrollable page, panel, list, table, or section.\n  - scrollDestinationDescription is required and describes what the scroll is trying to reveal or reach, using newly visible headings, section titles, list items, or stable content from the AFTER screenshot.\n  - Prefer descriptions like \"\u96C6\u6210\u5230 Playwright - Midscene - Vision-Driven UI Automation page, scrolling toward the API reference section\" or \"Android API documentation page, scrolling to the installation steps section\".\n  - Do NOT write generic phrases like \"more content\", \"the page\", \"current screen\", or \"main scrollable area\".\n- Scroll replayInstruction format: Scroll the page/region with description \"<elementDescription>\" by value \"<recorded value>\" until \"<scrollDestinationDescription>\" is visible.\n- Scroll actionSummary format: Scroll <elementDescription> toward <scrollDestinationDescription>.\n- Drag/Swipe rules:\n  - Drag replayInstruction format: Drag through the area described as \"<elementDescription>\".\n  - Swipe replayInstruction format: Swipe through the area described as \"<elementDescription>\".\n  - Describe start/end regions or the dragged UI control; do not describe only the gesture path.\n- KeyboardPress replayInstruction format: Press \"<value>\" on the element described as \"<elementDescription>\".\n- If uncertain, provide the best concrete visible text/role/context description. Set confidence to \"low\"; do not fall back to coordinates.";
+export declare const RECORDER_UI_DESCRIBER_SYSTEM_PROMPT = "You convert Studio preview recorder UI events into semantic replay instructions.\n\nThe recorder works from screenshots and mapped real-device coordinates only. Infer stable UI intent from the highlighted BEFORE screenshot. The AFTER screenshot is contextual evidence for state changes and scroll destinations.\n\nOutput JSON only:\n{\n  \"elementDescription\": \"short stable target/region description\",\n  \"replayInstruction\": \"one executable natural-language replay step\",\n  \"actionSummary\": \"short timeline summary\",\n  \"scrollDestinationDescription\": \"for scroll only: concrete newly visible destination content or goal\",\n  \"confidence\": \"high\" | \"medium\" | \"low\",\n  \"error\"?: \"only if no useful visual description can be inferred\"\n}\n\nRules:\n- Do NOT output coordinates as the main description.\n- Do NOT mention \"near coordinates\", \"nearby element\", \"near point\", \"red marker\", highlighted box, highlighted element, or screenshot.\n- Prefer stable target descriptions in this order: exact stable control text > stable label > role + stable section/context > icon purpose > visual position.\n- Treat placeholder or hint text that can change by user, time, data, or context as dynamic. Do not use dynamic hint values as the primary target description; prefer role + stable region + intent.\n- For repeated collections, treat item identity text as dynamic unless the user is clearly verifying that exact item. This includes any list/grid/table/feed/menu or repeated record surface. Do not output descriptions like \"<role> titled/named '<content>'\"; prefer stable role + region + selection intent.\n- Keep quoted UI text in the original UI language when it is a stable control label.\n- Apply the platform guidance from the user event:\n  - Web: button, input, link, menu item, tab, dialog, aria-label, placeholder, form section.\n  - Mobile: tab, list item, text field, icon button, navigation bar, bottom bar, sheet, card, screen section.\n  - Desktop/computer: menu item, toolbar button, dialog field, sidebar item, window control, file row, application region.\n- Pointer action rules:\n  - Preserve event.actionType semantics. Tap, DoubleClick, LongPress, and RightClick must not all become Click.\n  - Tap replayInstruction format: Tap on the element described as \"<elementDescription>\".\n  - DoubleClick replayInstruction format: Double click on the element described as \"<elementDescription>\".\n  - LongPress replayInstruction format: Long press the element described as \"<elementDescription>\".\n  - RightClick replayInstruction format: Right click on the element described as \"<elementDescription>\".\n  - Click replayInstruction format: Click on the element described as \"<elementDescription>\".\n- Input-specific rules:\n  - The highlighted BEFORE screenshot marks the field that receives the text.\n  - The screenshot after the action may show the typed value; use it only to confirm the field, never as the field description.\n  - elementDescription must identify the field itself by stable field role, nearby label, region, section, or sequence intent.\n  - Never use \"AI is analyzing element\", the typed value, the page title alone, or a generic \"input field\" as elementDescription.\n  - For consecutive input events, distinguish fields by stable role, section, order, current focus, filled/empty state, and neighboring actions instead of reusing the same generic field description.\n  - Input replayInstruction format: Input \"<value>\" into the element described as \"<elementDescription>\".\n- Scroll target quality bar:\n  - elementDescription describes the scrollable page, panel, list, table, or section at the highlighted scroll point.\n  - When multiple scrollable regions are visible, preserve the specific region where the scroll happened, such as left/right/top/bottom panel, navigation area, content pane, dialog body, table, list, or menu. Do not generalize a panel/list scroll into the whole page.\n  - scrollDestinationDescription is required and describes what the scroll is trying to reveal or reach, using newly visible headings, section titles, list items, or stable content from the AFTER screenshot.\n  - Prefer descriptions like \"Playwright integration documentation page, scrolling toward the API reference section\" or \"Android API documentation page, scrolling to the installation steps section\".\n  - Do NOT write generic phrases like \"more content\", \"the page\", \"current screen\", or \"main scrollable area\".\n- Scroll replayInstruction format: Scroll the page/region with description \"<elementDescription>\" by value \"<recorded value>\" until \"<scrollDestinationDescription>\" is visible.\n- Scroll actionSummary format: Scroll <elementDescription> toward <scrollDestinationDescription>.\n- Drag/Swipe rules:\n  - Drag replayInstruction format: Drag through the area described as \"<elementDescription>\".\n  - Swipe replayInstruction format: Swipe through the area described as \"<elementDescription>\".\n  - Describe start/end regions or the dragged UI control; do not describe only the gesture path.\n- KeyboardPress replayInstruction format: Press \"<value>\" on the element described as \"<elementDescription>\".\n- If uncertain, provide the best concrete visible text/role/context description. Set confidence to \"low\"; do not fall back to coordinates.";

package/dist/types/device/index.d.ts

@@ -153,6 +153,12 @@ export declare abstract class AbstractInterface {
      * is active.
      */
     flushPendingVisualUpdate?(): Promise<void>;
+    /**
+     * Optional non-blocking variant of `flushPendingVisualUpdate`. Keyboard-
+     * heavy preview interactions can schedule a coalesced refresh here without
+     * stalling the input hot path.
+     */
+    schedulePendingVisualUpdate?(): void;
     /**
      * Optional navigation state probe for browser-like interfaces, used to drive
      * loading indicators in playground UIs. Returning `undefined` means the

package/dist/types/service/index.d.ts

@@ -20,6 +20,7 @@ export default class Service {
     extract<T>(dataDemand: ServiceExtractParam, modelRuntime: ModelRuntime, opt?: ServiceExtractOption, pageDescription?: string, multimodalPrompt?: TMultimodalPrompt, context?: UIContext): Promise<ServiceExtractResult<T>>;
     describe(target: Rect | [number, number], modelRuntime: ModelRuntime, opt?: {
         deepLocate?: boolean;
+        context?: UIContext;
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;
 }
 export {};

package/dist/types/service/utils.d.ts

@@ -1,2 +1,4 @@
+import type { AIDescribeElementResponse } from '../types';
 import type { PartialServiceDumpFromSDK, ServiceDump } from '../types';
 export declare function createServiceDump(data: PartialServiceDumpFromSDK): ServiceDump;
+export declare function recoverDescribeResponseFromParseError(error: unknown): Pick<AIDescribeElementResponse, 'description'> | undefined;

package/dist/types/types.d.ts

@@ -79,6 +79,7 @@ export interface LocateValidatorResult {
     rect: Rect;
     center: [number, number];
     centerDistance?: number;
+    includedInRect?: boolean;
 }
 export interface AgentDescribeElementAtPointResult {
     prompt: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.9.6",
+  "version": "1.9.7",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -107,7 +107,7 @@
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "^3.25.1",
-    "@midscene/shared": "1.9.6"
+    "@midscene/shared": "1.9.7"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",