@midscene/core 0.26.3-beta-20250813105922.0 → 0.26.3-beta-20250813113027.0

package/dist/types/ai-model.d.ts

@@ -4,6 +4,7 @@ import { ChatCompletionMessageParam } from 'openai/resources/index';
 import type { ChatCompletionSystemMessageParam } from 'openai/resources/index';
 import type { ChatCompletionUserMessageParam } from 'openai/resources/index';
 import { ElementTreeNode } from '@midscene/shared/types';
+import type { NodeType } from '@midscene/shared/constants';
 import OpenAI from 'openai';
 import { Rect } from '@midscene/shared/types';
 import { Size } from '@midscene/shared/types';
@@ -206,7 +207,7 @@ declare interface DeviceAction<ParamType = any> {
     paramDescription?: string;
     location?: 'required' | 'optional' | false;
     whatToLocate?: string; // what to locate if location is required or optional
-    call: (param: ParamType) => Promise<void> | void;
+    call: (context: ExecutorContext, param: ParamType) => Promise<void> | void;
 }
 
 declare type ElementById = (id: string) => BaseElement | null;
@@ -219,6 +220,85 @@ export declare function elementByPositionWithElementInfo(treeRoot: ElementTreeNo
     filterPositionElements?: boolean;
 }): BaseElement | undefined;
 
+declare interface ExecutionRecorderItem {
+    type: 'screenshot';
+    ts: number;
+    screenshot?: string;
+    timing?: string;
+}
+
+declare type ExecutionTask<
+E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<
+any,
+any,
+any
+>,
+> = E &
+ExecutionTaskReturn<
+E extends ExecutionTaskApply<any, any, infer TaskOutput, any>
+? TaskOutput
+: unknown,
+E extends ExecutionTaskApply<any, any, any, infer TaskLog>
+? TaskLog
+: unknown
+> & {
+    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';
+    error?: Error;
+    errorMessage?: string;
+    errorStack?: string;
+    timing?: {
+        start: number;
+        end?: number;
+        cost?: number;
+    };
+    usage?: AIUsageInfo;
+};
+
+declare interface ExecutionTaskApply<
+Type extends ExecutionTaskType = any,
+TaskParam = any,
+TaskOutput = any,
+TaskLog = any,
+> {
+    type: Type;
+    subType?: string;
+    param?: TaskParam;
+    thought?: string;
+    locate?: PlanningLocateParam | null;
+    pageContext?: UIContext;
+    executor: (
+    param: TaskParam,
+    context: ExecutorContext,
+    ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>
+    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>
+    | undefined
+    | void;
+}
+
+declare interface ExecutionTaskHitBy {
+    from: string;
+    context: Record<string, any>;
+}
+
+declare interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {
+    output?: TaskOutput;
+    log?: TaskLog;
+    recorder?: ExecutionRecorderItem[];
+    hitBy?: ExecutionTaskHitBy;
+}
+
+declare type ExecutionTaskType =
+| 'Planning'
+| 'Insight'
+| 'Action'
+| 'Assertion'
+| 'Log';
+
+declare interface ExecutorContext {
+    task: ExecutionTask;
+    element?: LocateResultElement | null;
+}
+
 /**
  * Generates Playwright test code from recorded events
  */
@@ -252,6 +332,19 @@ declare interface LocateOption {
     pageContext?: UIContext<BaseElement>;
 }
 
+declare type LocateResultElement = {
+    id: string;
+    indexId?: number;
+    center: [number, number];
+    rect: Rect;
+    xpaths: string[];
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    isOrderSensitive?: boolean;
+};
+
 declare type MidsceneYamlFlowItem =
 | MidsceneYamlFlowItemAIAction
 | MidsceneYamlFlowItemAIAssert
@@ -303,7 +396,7 @@ declare interface MidsceneYamlFlowItemAIRightClick extends LocateOption {
 
 declare interface MidsceneYamlFlowItemAIScroll
 extends LocateOption,
-PlanningActionParamScroll {
+ScrollParam {
     aiScroll: null;
     locate?: TUserPrompt; // which area to scroll, optional
 }
@@ -367,8 +460,6 @@ declare interface PlanningAction<ParamType = any> {
     locate?: PlanningLocateParam | null;
 }
 
-declare type PlanningActionParamScroll = scrollParam;
-
 declare interface PlanningAIResponse {
     action?: PlanningAction; // this is the qwen mode
     actions?: PlanningAction[];
@@ -413,7 +504,7 @@ declare interface ReferenceImage {
 
 export declare function resizeImageForUiTars(imageBase64: string, size: Size): Promise<string>;
 
-declare interface scrollParam {
+declare interface ScrollParam {
     direction: 'down' | 'up' | 'right' | 'left';
     scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
     distance?: null | number; // distance in px

package/dist/types/index.d.ts

@@ -208,6 +208,16 @@ declare type AIUsageInfo_2 = Record<string, any> & {
     time_cost: number | undefined;
 };
 
+export declare interface AndroidLongPressParam {
+    duration?: number;
+}
+
+export declare interface AndroidPullParam {
+    direction: 'up' | 'down';
+    distance?: number;
+    duration?: number;
+}
+
 export declare interface BaseAgentParserOpt {
     selector?: string;
     ignoreMarker?: boolean;
@@ -308,7 +318,7 @@ export declare interface DeviceAction<ParamType = any> {
     paramDescription?: string;
     location?: 'required' | 'optional' | false;
     whatToLocate?: string;
-    call: (param: ParamType) => Promise<void> | void;
+    call: (context: ExecutorContext, param: ParamType) => Promise<void> | void;
 }
 
 declare interface DeviceAction_2<ParamType = any> {
@@ -318,7 +328,7 @@ declare interface DeviceAction_2<ParamType = any> {
     paramDescription?: string;
     location?: 'required' | 'optional' | false;
     whatToLocate?: string; // what to locate if location is required or optional
-    call: (param: ParamType) => Promise<void> | void;
+    call: (context: ExecutorContext_2, param: ParamType) => Promise<void> | void;
 }
 
 export declare interface DumpMeta {
@@ -908,14 +918,14 @@ declare interface MidsceneYamlFlowItemAIRightClick_2 extends LocateOption_2 {
     aiRightClick: TUserPrompt_2;
 }
 
-export declare interface MidsceneYamlFlowItemAIScroll extends LocateOption, PlanningActionParamScroll {
+export declare interface MidsceneYamlFlowItemAIScroll extends LocateOption, ScrollParam {
     aiScroll: null;
     locate?: TUserPrompt;
 }
 
 declare interface MidsceneYamlFlowItemAIScroll_2
 extends LocateOption_2,
-PlanningActionParamScroll_2 {
+ScrollParam_2 {
     aiScroll: null;
     locate?: TUserPrompt_2; // which area to scroll, optional
 }
@@ -1067,22 +1077,6 @@ declare interface PlanningAction_2<ParamType = any> {
     locate?: PlanningLocateParam_2 | null;
 }
 
-export declare interface PlanningActionParamAndroidLongPress {
-    x: number;
-    y: number;
-    duration?: number;
-}
-
-export declare interface PlanningActionParamAndroidPull {
-    direction: 'up' | 'down';
-    startPoint?: {
-        x: number;
-        y: number;
-    };
-    distance?: number;
-    duration?: number;
-}
-
 export declare interface PlanningActionParamAssert {
     assertion: TUserPrompt;
 }
@@ -1100,10 +1094,6 @@ export declare interface PlanningActionParamInputOrKeyPress {
 
 export declare type PlanningActionParamRightClick = null;
 
-export declare type PlanningActionParamScroll = scrollParam;
-
-declare type PlanningActionParamScroll_2 = scrollParam_2;
-
 export declare interface PlanningActionParamSleep {
     timeMs: number;
 }
@@ -1193,13 +1183,13 @@ export declare interface ScriptPlayerTaskStatus extends MidsceneYamlTask {
     error?: Error;
 }
 
-export declare interface scrollParam {
+export declare interface ScrollParam {
     direction: 'down' | 'up' | 'right' | 'left';
     scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
     distance?: null | number;
 }
 
-declare interface scrollParam_2 {
+declare interface ScrollParam_2 {
     direction: 'down' | 'up' | 'right' | 'left';
     scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
     distance?: null | number; // distance in px

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "0.26.3-beta-20250813105922.0",
+  "version": "0.26.3-beta-20250813113027.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -60,8 +60,8 @@
     "langsmith": "0.3.7",
     "openai": "4.81.0",
     "socks-proxy-agent": "8.0.4",
-    "@midscene/recorder": "0.26.3-beta-20250813105922.0",
-    "@midscene/shared": "0.26.3-beta-20250813105922.0"
+    "@midscene/shared": "0.26.3-beta-20250813113027.0",
+    "@midscene/recorder": "0.26.3-beta-20250813113027.0"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "^7.52.10",