@midscene/core 1.8.0 → 1.8.1

package/dist/types/agent/usage-intent.d.ts

@@ -0,0 +1,3 @@
+import type { AIUsageInfo } from '../types';
+import type { TIntent } from '@midscene/shared/env';
+export declare function withUsageIntent(usage: AIUsageInfo | undefined, intent: TIntent): AIUsageInfo | undefined;

package/dist/types/device/index.d.ts

@@ -19,6 +19,97 @@ export interface MjpegStreamOptions {
     onFrame(frame: MjpegStreamFrame): void;
     onError?(error: unknown): void;
 }
+/** A point in device-pixel coordinates on the screen. */
+export interface PointerPoint {
+    x: number;
+    y: number;
+}
+export interface PointerInputPrimitives {
+    tap(p: PointerPoint, opts?: {
+        duration?: number;
+    }): Promise<void>;
+    doubleClick?(p: PointerPoint): Promise<void>;
+    rightClick?(p: PointerPoint): Promise<void>;
+    hover?(p: PointerPoint): Promise<void>;
+    longPress?(p: PointerPoint, opts?: {
+        duration?: number;
+    }): Promise<void>;
+    dragAndDrop?(from: PointerPoint, to: PointerPoint): Promise<void>;
+}
+export interface TouchInputPrimitives {
+    swipe(start: PointerPoint, end: PointerPoint, opts?: {
+        duration?: number;
+        repeat?: number;
+    }): Promise<void>;
+    pinch?(center: PointerPoint, opts: {
+        startDistance: number;
+        endDistance: number;
+        duration: number;
+    }): Promise<void>;
+}
+export interface KeyboardInputPrimitives {
+    keyboardPress(keyName: string, opts?: {
+        target?: unknown;
+    }): Promise<void>;
+    cursorMove?(direction: 'left' | 'right', times?: number): Promise<void>;
+    typeText(value: string, opts?: {
+        autoDismissKeyboard?: boolean;
+        target?: unknown;
+        replace?: boolean;
+        focusOnly?: boolean;
+    }): Promise<void>;
+    clearInput(target?: unknown): Promise<void>;
+}
+export interface ScrollInputPrimitives {
+    scroll(param: ActionScrollParam): Promise<void>;
+}
+export interface SystemInputPrimitives {
+    backButton?(): Promise<void>;
+    homeButton?(): Promise<void>;
+    recentAppsButton?(): Promise<void>;
+}
+export interface InputPrimitives {
+    pointer?: PointerInputPrimitives;
+    keyboard?: KeyboardInputPrimitives;
+    touch?: TouchInputPrimitives;
+    scroll?: ScrollInputPrimitives;
+    system?: SystemInputPrimitives;
+}
+export interface MobileInputPrimitives extends InputPrimitives {
+    pointer: PointerInputPrimitives & {
+        doubleClick(p: PointerPoint): Promise<void>;
+        longPress(p: PointerPoint, opts?: {
+            duration?: number;
+        }): Promise<void>;
+        dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;
+    };
+    keyboard: KeyboardInputPrimitives;
+    touch: TouchInputPrimitives;
+}
+export interface BrowserInputPrimitives extends InputPrimitives {
+    pointer: PointerInputPrimitives & {
+        doubleClick(p: PointerPoint): Promise<void>;
+        rightClick(p: PointerPoint): Promise<void>;
+        hover(p: PointerPoint): Promise<void>;
+        dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;
+        longPress(p: PointerPoint, opts?: {
+            duration?: number;
+        }): Promise<void>;
+    };
+    keyboard: KeyboardInputPrimitives;
+    scroll: ScrollInputPrimitives;
+    touch: TouchInputPrimitives;
+}
+export interface ComputerInputPrimitives extends InputPrimitives {
+    pointer: PointerInputPrimitives & {
+        doubleClick(p: PointerPoint): Promise<void>;
+        rightClick(p: PointerPoint): Promise<void>;
+        hover(p: PointerPoint): Promise<void>;
+        dragAndDrop(from: PointerPoint, to: PointerPoint): Promise<void>;
+    };
+    keyboard: KeyboardInputPrimitives;
+    scroll: ScrollInputPrimitives;
+}
 export declare abstract class AbstractInterface {
     abstract interfaceType: string;
     abstract screenshotBase64(): Promise<string>;
@@ -68,6 +159,12 @@ export declare abstract class AbstractInterface {
     navigationState?(): Promise<{
         isLoading: boolean;
     }>;
+    /**
+     * Low-level device input surface. Platform implementations expose transport
+     * primitives here; higher-level AI actions and manual pointer dispatch should
+     * adapt to this instead of duplicating platform gesture logic.
+     */
+    inputPrimitives?: InputPrimitives;
 }
 export declare const defineAction: <TSchema extends z.ZodType | undefined = undefined, TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined, TReturn = any>(config: {
     name: string;
@@ -226,7 +323,7 @@ export declare const actionTapParamSchema: z.ZodObject<{
 export type ActionTapParam = {
     locate: LocateResultElement;
 };
-export declare const defineActionTap: (call: (param: ActionTapParam) => Promise<void>) => DeviceAction<ActionTapParam>;
+export declare const defineActionTap: (tap: PointerInputPrimitives["tap"]) => DeviceAction<ActionTapParam>;
 export declare const actionRightClickParamSchema: z.ZodObject<{
     locate: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -377,7 +474,7 @@ export declare const actionRightClickParamSchema: z.ZodObject<{
 export type ActionRightClickParam = {
     locate: LocateResultElement;
 };
-export declare const defineActionRightClick: (call: (param: ActionRightClickParam) => Promise<void>) => DeviceAction<ActionRightClickParam>;
+export declare const defineActionRightClick: (rightClick: NonNullable<PointerInputPrimitives["rightClick"]>) => DeviceAction<ActionRightClickParam>;
 export declare const actionDoubleClickParamSchema: z.ZodObject<{
     locate: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -528,7 +625,7 @@ export declare const actionDoubleClickParamSchema: z.ZodObject<{
 export type ActionDoubleClickParam = {
     locate: LocateResultElement;
 };
-export declare const defineActionDoubleClick: (call: (param: ActionDoubleClickParam) => Promise<void>) => DeviceAction<ActionDoubleClickParam>;
+export declare const defineActionDoubleClick: (doubleClick: NonNullable<PointerInputPrimitives["doubleClick"]>) => DeviceAction<ActionDoubleClickParam>;
 export declare const actionHoverParamSchema: z.ZodObject<{
     locate: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -679,7 +776,7 @@ export declare const actionHoverParamSchema: z.ZodObject<{
 export type ActionHoverParam = {
     locate: LocateResultElement;
 };
-export declare const defineActionHover: (call: (param: ActionHoverParam) => Promise<void>) => DeviceAction<ActionHoverParam>;
+export declare const defineActionHover: (hover: NonNullable<PointerInputPrimitives["hover"]>) => DeviceAction<ActionHoverParam>;
 export declare const actionInputParamSchema: z.ZodObject<{
     value: z.ZodEffects<z.ZodUnion<[z.ZodString, z.ZodNumber]>, string, string | number>;
     locate: z.ZodOptional<z.ZodObject<{
@@ -792,6 +889,7 @@ export declare const actionInputParamSchema: z.ZodObject<{
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough">>>;
     mode: z.ZodDefault<z.ZodEnum<["replace", "clear", "typeOnly"]>>;
+    autoDismissKeyboard: z.ZodOptional<z.ZodBoolean>;
 }, "strip", z.ZodTypeAny, {
     value: string;
     mode: "replace" | "clear" | "typeOnly";
@@ -832,6 +930,7 @@ export declare const actionInputParamSchema: z.ZodObject<{
         cacheable: z.ZodOptional<z.ZodBoolean>;
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
+    autoDismissKeyboard?: boolean | undefined;
 }, {
     value: string | number;
     locate?: z.objectInputType<{
@@ -872,13 +971,15 @@ export declare const actionInputParamSchema: z.ZodObject<{
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
     mode?: "replace" | "clear" | "typeOnly" | undefined;
+    autoDismissKeyboard?: boolean | undefined;
 }>;
 export type ActionInputParam = {
     value: string;
     locate?: LocateResultElement;
     mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
+    autoDismissKeyboard?: boolean;
 };
-export declare const defineActionInput: (call: (param: ActionInputParam) => Promise<void>) => DeviceAction<ActionInputParam>;
+export declare const defineActionInput: (keyboard: KeyboardInputPrimitives) => DeviceAction<ActionInputParam>;
 export declare const actionKeyboardPressParamSchema: z.ZodObject<{
     locate: z.ZodOptional<z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1073,7 +1174,7 @@ export type ActionKeyboardPressParam = {
     locate?: LocateResultElement;
     keyName: string;
 };
-export declare const defineActionKeyboardPress: (call: (param: ActionKeyboardPressParam) => Promise<void>) => DeviceAction<ActionKeyboardPressParam>;
+export declare const defineActionKeyboardPress: (keyboardPress: KeyboardInputPrimitives["keyboardPress"]) => DeviceAction<ActionKeyboardPressParam>;
 export declare const actionScrollParamSchema: z.ZodObject<{
     scrollType: z.ZodDefault<z.ZodEnum<["singleAction", "scrollToBottom", "scrollToTop", "scrollToRight", "scrollToLeft"]>>;
     direction: z.ZodDefault<z.ZodEnum<["down", "up", "right", "left"]>>;
@@ -1189,7 +1290,7 @@ export declare const actionScrollParamSchema: z.ZodObject<{
     }, z.ZodTypeAny, "passthrough">>>;
 }, "strip", z.ZodTypeAny, {
     scrollType: "singleAction" | "scrollToBottom" | "scrollToTop" | "scrollToRight" | "scrollToLeft";
-    direction: "down" | "up" | "right" | "left";
+    direction: "left" | "right" | "down" | "up";
     locate?: z.objectOutputType<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
             prompt: z.ZodString;
@@ -1267,10 +1368,10 @@ export declare const actionScrollParamSchema: z.ZodObject<{
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
     scrollType?: "singleAction" | "scrollToBottom" | "scrollToTop" | "scrollToRight" | "scrollToLeft" | undefined;
-    direction?: "down" | "up" | "right" | "left" | undefined;
+    direction?: "left" | "right" | "down" | "up" | undefined;
     distance?: number | null | undefined;
 }>;
-export declare const defineActionScroll: (call: (param: ActionScrollParam) => Promise<void>) => DeviceAction<ActionScrollParam>;
+export declare const defineActionScroll: (scroll: ScrollInputPrimitives["scroll"]) => DeviceAction<ActionScrollParam>;
 export declare const actionDragAndDropParamSchema: z.ZodObject<{
     from: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1565,7 +1666,7 @@ export type ActionDragAndDropParam = {
     from: LocateResultElement;
     to: LocateResultElement;
 };
-export declare const defineActionDragAndDrop: (call: (param: ActionDragAndDropParam) => Promise<void>) => DeviceAction<ActionDragAndDropParam>;
+export declare const defineActionDragAndDrop: (dragAndDrop: NonNullable<PointerInputPrimitives["dragAndDrop"]>) => DeviceAction<ActionDragAndDropParam>;
 export declare const ActionLongPressParamSchema: z.ZodObject<{
     locate: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1720,7 +1821,7 @@ export type ActionLongPressParam = {
     locate: LocateResultElement;
     duration?: number;
 };
-export declare const defineActionLongPress: (call: (param: ActionLongPressParam) => Promise<void>) => DeviceAction<ActionLongPressParam>;
+export declare const defineActionLongPress: (longPress: NonNullable<PointerInputPrimitives["longPress"]>) => DeviceAction<ActionLongPressParam>;
 export declare const ActionSwipeParamSchema: z.ZodObject<{
     start: z.ZodOptional<z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1946,7 +2047,7 @@ export declare const ActionSwipeParamSchema: z.ZodObject<{
     repeat: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     duration: number;
-    direction?: "down" | "up" | "right" | "left" | undefined;
+    direction?: "left" | "right" | "down" | "up" | undefined;
     distance?: number | undefined;
     start?: z.objectOutputType<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -2024,7 +2125,7 @@ export declare const ActionSwipeParamSchema: z.ZodObject<{
     }, z.ZodTypeAny, "passthrough"> | undefined;
     repeat?: number | undefined;
 }, {
-    direction?: "down" | "up" | "right" | "left" | undefined;
+    direction?: "left" | "right" | "down" | "up" | undefined;
     distance?: number | undefined;
     duration?: number | undefined;
     start?: z.objectInputType<{
@@ -2126,7 +2227,10 @@ export declare function normalizeMobileSwipeParam(param: ActionSwipeParam, scree
     duration: number;
     repeatCount: number;
 };
-export declare const defineActionSwipe: (call: (param: ActionSwipeParam) => Promise<void>) => DeviceAction<ActionSwipeParam>;
+export declare const defineActionSwipe: (config: {
+    swipe: TouchInputPrimitives["swipe"];
+    size(): Promise<Size>;
+}) => DeviceAction<ActionSwipeParam>;
 export declare const actionClearInputParamSchema: z.ZodObject<{
     locate: z.ZodOptional<z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -2317,22 +2421,25 @@ export declare const actionClearInputParamSchema: z.ZodObject<{
 export type ActionClearInputParam = {
     locate?: LocateResultElement;
 };
-export declare const defineActionClearInput: (call: (param: ActionClearInputParam) => Promise<void>) => DeviceAction<ActionClearInputParam>;
+export declare const defineActionClearInput: (clearInput: KeyboardInputPrimitives["clearInput"]) => DeviceAction<ActionClearInputParam>;
 export declare const actionCursorMoveParamSchema: z.ZodObject<{
     direction: z.ZodEnum<["left", "right"]>;
     times: z.ZodDefault<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
-    direction: "right" | "left";
+    direction: "left" | "right";
     times: number;
 }, {
-    direction: "right" | "left";
+    direction: "left" | "right";
     times?: number | undefined;
 }>;
 export type ActionCursorMoveParam = {
     direction: 'left' | 'right';
     times?: number;
 };
-export declare const defineActionCursorMove: (call: (param: ActionCursorMoveParam) => Promise<void>) => DeviceAction<ActionCursorMoveParam>;
+export declare const defineActionCursorMove: (config: {
+    keyboard: Pick<KeyboardInputPrimitives, "keyboardPress" | "cursorMove">;
+    sleep?(timeMs: number): Promise<void>;
+}) => DeviceAction<ActionCursorMoveParam>;
 export declare const ActionPinchParamSchema: z.ZodObject<{
     locate: z.ZodOptional<z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -2535,7 +2642,10 @@ export type ActionPinchParam = {
     distance?: number;
     duration?: number;
 };
-export declare const defineActionPinch: (call: (param: ActionPinchParam) => Promise<void>) => DeviceAction<ActionPinchParam>;
+export declare const defineActionPinch: (config: {
+    pinch: TouchInputPrimitives["pinch"];
+    size(): Promise<Size>;
+}) => DeviceAction<ActionPinchParam> | undefined;
 export declare function normalizePinchParam(param: ActionPinchParam, screenSize: {
     width: number;
     height: number;
@@ -2546,6 +2656,34 @@ export declare function normalizePinchParam(param: ActionPinchParam, screenSize:
     endDistance: number;
     duration: number;
 };
+export interface MobileInputActionContext {
+    input: MobileInputPrimitives;
+    size(): Promise<Size>;
+    sleep?(timeMs: number): Promise<void>;
+    getDefaultAutoDismissKeyboard?(): boolean | undefined;
+    systemActions?: SystemInputActionOptions;
+}
+export interface SystemInputActionConfig {
+    name: string;
+    description: string;
+    interfaceAlias?: string;
+    delayBeforeRunner?: number;
+    delayAfterRunner?: number;
+}
+export interface SystemInputActionOptions {
+    backButton?: SystemInputActionConfig;
+    homeButton?: SystemInputActionConfig;
+    recentAppsButton?: SystemInputActionConfig;
+}
+export interface InputPrimitiveActionOptions {
+    size?: () => Promise<Size>;
+    sleep?: (timeMs: number) => Promise<void>;
+    includeSwipe?: boolean;
+    includePinch?: boolean;
+    systemActions?: SystemInputActionOptions;
+}
+export declare function defineActionsFromInputPrimitives(input: InputPrimitives, options?: InputPrimitiveActionOptions): DeviceAction<any>[];
+export declare function createDefaultMobileActions(context: MobileInputActionContext): DeviceAction<any>[];
 export declare const ActionSleepParamSchema: z.ZodObject<{
     timeMs: z.ZodOptional<z.ZodDefault<z.ZodNumber>>;
 }, "strip", z.ZodTypeAny, {

package/dist/types/types.d.ts

@@ -15,7 +15,15 @@ export type AIUsageInfo = Record<string, any> & {
     time_cost: number | undefined;
     model_name: string | undefined;
     model_description: string | undefined;
+    /**
+     * Semantic intent of the model call, such as default, planning, or insight.
+     */
     intent: string | undefined;
+    /**
+     * Config slot where the model config was resolved from. For example, a
+     * planning call may use the default slot when no planning model is configured.
+     */
+    slot: string | undefined;
     request_id: string | undefined;
 };
 export type { LocateResultElement };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.8.0",
+  "version": "1.8.1",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -97,7 +97,7 @@
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "^3.25.1",
-    "@midscene/shared": "1.8.0"
+    "@midscene/shared": "1.8.1"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",