@midscene/core 0.30.10 → 1.0.0

package/dist/types/device/index.d.ts

@@ -1,4 +1,5 @@
-import type { DeviceAction, LocateResultElement } from '../types';
+import type { ActionScrollParam, DeviceAction, LocateResultElement } from '../types';
+import type { IModelConfig } from '@midscene/shared/env';
 import type { ElementNode } from '@midscene/shared/extractor';
 import { z } from 'zod';
 import type { ElementCacheFeature, Rect, Size, UIContext } from '../types';
@@ -6,9 +7,10 @@ export declare abstract class AbstractInterface {
     abstract interfaceType: string;
     abstract screenshotBase64(): Promise<string>;
     abstract size(): Promise<Size>;
-    abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;
-    abstract cacheFeatureForRect?(rect: Rect, opt?: {
-        _orderSensitive: boolean;
+    abstract actionSpace(): DeviceAction[];
+    abstract cacheFeatureForRect?(rect: Rect, options?: {
+        targetDescription?: string;
+        modelConfig?: IModelConfig;
     }): Promise<ElementCacheFeature>;
     abstract rectMatchesCacheFeature?(feature: ElementCacheFeature): Promise<Rect>;
     abstract destroy?(): Promise<void>;
@@ -20,13 +22,13 @@ export declare abstract class AbstractInterface {
     abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;
     abstract getContext?(): Promise<UIContext>;
 }
-export declare const defineAction: <TSchema extends z.ZodType, TRuntime = z.infer<TSchema>>(config: {
+export declare const defineAction: <TSchema extends z.ZodType | undefined = undefined, TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined, TReturn = any>(config: {
     name: string;
     description: string;
     interfaceAlias?: string;
-    paramSchema: TSchema;
-    call: (param: TRuntime) => Promise<void>;
-} & Partial<Omit<DeviceAction<TRuntime>, "name" | "description" | "interfaceAlias" | "paramSchema" | "call">>) => DeviceAction<TRuntime>;
+    paramSchema?: TSchema;
+    call: (param: TRuntime) => Promise<TReturn> | TReturn;
+} & Partial<Omit<DeviceAction<TRuntime, TReturn>, "name" | "description" | "interfaceAlias" | "paramSchema" | "call">>) => DeviceAction<TRuntime, TReturn>;
 export declare const actionTapParamSchema: z.ZodObject<{
     locate: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -145,8 +147,8 @@ export declare const actionTapParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -162,8 +164,8 @@ export declare const actionTapParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -291,8 +293,8 @@ export declare const actionRightClickParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -308,8 +310,8 @@ export declare const actionRightClickParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -437,8 +439,8 @@ export declare const actionDoubleClickParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -454,8 +456,8 @@ export declare const actionDoubleClickParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -583,8 +585,8 @@ export declare const actionHoverParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -600,8 +602,8 @@ export declare const actionHoverParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -997,7 +999,7 @@ export type ActionKeyboardPressParam = {
 export declare const defineActionKeyboardPress: (call: (param: ActionKeyboardPressParam) => Promise<void>) => DeviceAction<ActionKeyboardPressParam>;
 export declare const actionScrollParamSchema: z.ZodObject<{
     direction: z.ZodDefault<z.ZodEnum<["down", "up", "right", "left"]>>;
-    scrollType: z.ZodDefault<z.ZodEnum<["once", "untilBottom", "untilTop", "untilRight", "untilLeft"]>>;
+    scrollType: z.ZodDefault<z.ZodEnum<["singleAction", "scrollToBottom", "scrollToTop", "scrollToRight", "scrollToLeft"]>>;
     distance: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
     locate: z.ZodOptional<z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1106,8 +1108,8 @@ export declare const actionScrollParamSchema: z.ZodObject<{
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough">>>;
 }, "strip", z.ZodTypeAny, {
-    direction: "left" | "right" | "down" | "up";
-    scrollType: "once" | "untilBottom" | "untilTop" | "untilRight" | "untilLeft";
+    direction: "down" | "up" | "right" | "left";
+    scrollType: "singleAction" | "scrollToBottom" | "scrollToTop" | "scrollToRight" | "scrollToLeft";
     locate?: z.objectOutputType<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
             prompt: z.ZodString;
@@ -1182,16 +1184,10 @@ export declare const actionScrollParamSchema: z.ZodObject<{
         cacheable: z.ZodOptional<z.ZodBoolean>;
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
-    direction?: "left" | "right" | "down" | "up" | undefined;
-    scrollType?: "once" | "untilBottom" | "untilTop" | "untilRight" | "untilLeft" | undefined;
+    direction?: "down" | "up" | "right" | "left" | undefined;
+    scrollType?: "singleAction" | "scrollToBottom" | "scrollToTop" | "scrollToRight" | "scrollToLeft" | undefined;
     distance?: number | null | undefined;
 }>;
-export type ActionScrollParam = {
-    direction?: 'down' | 'up' | 'right' | 'left';
-    scrollType?: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
-    distance?: number | null;
-    locate?: LocateResultElement;
-};
 export declare const defineActionScroll: (call: (param: ActionScrollParam) => Promise<void>) => DeviceAction<ActionScrollParam>;
 export declare const actionDragAndDropParamSchema: z.ZodObject<{
     from: z.ZodObject<{
@@ -1417,8 +1413,8 @@ export declare const actionDragAndDropParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -1433,8 +1429,8 @@ export declare const actionDragAndDropParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -1450,8 +1446,8 @@ export declare const actionDragAndDropParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -1466,8 +1462,8 @@ export declare const actionDragAndDropParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -1597,8 +1593,8 @@ export declare const ActionLongPressParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -1615,8 +1611,8 @@ export declare const ActionLongPressParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -1847,8 +1843,7 @@ export declare const ActionSwipeParamSchema: z.ZodObject<{
     repeat: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     duration: number;
-    repeat?: number | undefined;
-    direction?: "left" | "right" | "down" | "up" | undefined;
+    direction?: "down" | "up" | "right" | "left" | undefined;
     distance?: number | undefined;
     start?: z.objectOutputType<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{
@@ -1922,9 +1917,9 @@ export declare const ActionSwipeParamSchema: z.ZodObject<{
         cacheable: z.ZodOptional<z.ZodBoolean>;
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
-}, {
     repeat?: number | undefined;
-    direction?: "left" | "right" | "down" | "up" | undefined;
+}, {
+    direction?: "down" | "up" | "right" | "left" | undefined;
     distance?: number | undefined;
     duration?: number | undefined;
     start?: z.objectInputType<{
@@ -1999,6 +1994,7 @@ export declare const ActionSwipeParamSchema: z.ZodObject<{
         cacheable: z.ZodOptional<z.ZodBoolean>;
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
+    repeat?: number | undefined;
 }>;
 export type ActionSwipeParam = {
     start?: LocateResultElement;
@@ -2127,8 +2123,8 @@ export declare const actionClearInputParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -2144,8 +2140,8 @@ export declare const actionClearInputParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        cacheable?: boolean | undefined;
         deepThink?: boolean | undefined;
+        cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
         [k: string]: unknown;
@@ -2155,4 +2151,24 @@ export type ActionClearInputParam = {
     locate: LocateResultElement;
 };
 export declare const defineActionClearInput: (call: (param: ActionClearInputParam) => Promise<void>) => DeviceAction<ActionClearInputParam>;
+export declare const actionAssertParamSchema: z.ZodObject<{
+    condition: z.ZodString;
+    thought: z.ZodString;
+    result: z.ZodBoolean;
+}, "strip", z.ZodTypeAny, {
+    condition: string;
+    thought: string;
+    result: boolean;
+}, {
+    condition: string;
+    thought: string;
+    result: boolean;
+}>;
+export type ActionAssertParam = {
+    condition: string;
+    thought: string;
+    result: boolean;
+};
+export declare const defineActionAssert: () => DeviceAction<ActionAssertParam>;
 export type { DeviceAction } from '../types';
+export type { AndroidDeviceOpt, AndroidDeviceInputOpt, IOSDeviceOpt, IOSDeviceInputOpt, } from './device-options';

package/dist/types/index.d.ts

@@ -1,12 +1,13 @@
 import { z } from 'zod';
-import { Executor } from './ai-model/action-executor';
-import Insight from './insight/index';
+import Service from './service/index';
+import { TaskRunner } from './task-runner';
 import { getVersion } from './utils';
 export { plan, describeUserPage, AiLocateElement, getMidsceneLocationSchema, type MidsceneLocationResultType, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, } from './ai-model/index';
-export { MIDSCENE_MODEL_NAME } from '@midscene/shared/env';
+export { MIDSCENE_MODEL_NAME, type CreateOpenAIClientFn, } from '@midscene/shared/env';
 export type * from './types';
+export { ServiceError } from './types';
 export { z };
-export default Insight;
-export { Executor, Insight, getVersion };
+export default Service;
+export { TaskRunner, Service, getVersion };
 export type { MidsceneYamlScript, MidsceneYamlTask, MidsceneYamlFlowItem, MidsceneYamlConfigResult, MidsceneYamlConfig, MidsceneYamlScriptWebEnv, MidsceneYamlScriptAndroidEnv, MidsceneYamlScriptIOSEnv, MidsceneYamlScriptEnv, LocateOption, DetailedLocateParam, } from './yaml';
-export { Agent, type AgentOpt, createAgent } from './agent';
+export { Agent, type AgentOpt, type AiActOptions, createAgent } from './agent';

package/dist/types/service/index.d.ts

@@ -0,0 +1,26 @@
+import { callAIWithObjectResponse } from '../ai-model/index';
+import type { AIDescribeElementResponse, DetailedLocateParam, LocateResultWithDump, Rect, ServiceExtractOption, ServiceExtractParam, ServiceExtractResult, ServiceTaskInfo, UIContext } from '../types';
+import { type IModelConfig } from '@midscene/shared/env';
+import type { TMultimodalPrompt } from '../common';
+export interface LocateOpts {
+    context?: UIContext;
+}
+export type AnyValue<T> = {
+    [K in keyof T]: unknown extends T[K] ? any : T[K];
+};
+interface ServiceOptions {
+    taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;
+    aiVendorFn?: typeof callAIWithObjectResponse;
+}
+export default class Service {
+    contextRetrieverFn: () => Promise<UIContext> | UIContext;
+    aiVendorFn: Exclude<ServiceOptions['aiVendorFn'], undefined>;
+    taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;
+    constructor(context: UIContext | (() => Promise<UIContext> | UIContext), opt?: ServiceOptions);
+    locate(query: DetailedLocateParam, opt: LocateOpts, modelConfig: IModelConfig): Promise<LocateResultWithDump>;
+    extract<T>(dataDemand: ServiceExtractParam, modelConfig: IModelConfig, opt?: ServiceExtractOption, pageDescription?: string, multimodalPrompt?: TMultimodalPrompt): Promise<ServiceExtractResult<T>>;
+    describe(target: Rect | [number, number], modelConfig: IModelConfig, opt?: {
+        deepThink?: boolean;
+    }): Promise<Pick<AIDescribeElementResponse, 'description'>>;
+}
+export {};

package/dist/types/service/utils.d.ts

@@ -0,0 +1,2 @@
+import type { PartialServiceDumpFromSDK, ServiceDump } from '../types';
+export declare function createServiceDump(data: PartialServiceDumpFromSDK): ServiceDump;

package/dist/types/task-runner.d.ts

@@ -0,0 +1,49 @@
+import type { ExecutionDump, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions, UIContext } from './types';
+type TaskRunnerInitOptions = ExecutionTaskProgressOptions & {
+    tasks?: ExecutionTaskApply[];
+    onTaskUpdate?: (runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void;
+};
+type TaskRunnerOperationOptions = {
+    allowWhenError?: boolean;
+};
+export declare class TaskRunner {
+    name: string;
+    tasks: ExecutionTask[];
+    status: 'init' | 'pending' | 'running' | 'completed' | 'error';
+    onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
+    private readonly uiContextBuilder;
+    private readonly onTaskUpdate?;
+    constructor(name: string, uiContextBuilder: () => Promise<UIContext>, options?: TaskRunnerInitOptions);
+    private emitOnTaskUpdate;
+    private lastUiContext?;
+    private getUiContext;
+    private captureScreenshot;
+    private attachRecorderItem;
+    private markTaskAsPending;
+    private normalizeStatusFromError;
+    private findPreviousNonSubTaskUIContext;
+    append(task: ExecutionTaskApply[] | ExecutionTaskApply, options?: TaskRunnerOperationOptions): Promise<void>;
+    appendAndFlush(task: ExecutionTaskApply[] | ExecutionTaskApply, options?: TaskRunnerOperationOptions): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
+    flush(options?: TaskRunnerOperationOptions): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
+    isInErrorState(): boolean;
+    latestErrorTask(): ExecutionTask | null;
+    dump(): ExecutionDump;
+    appendErrorPlan(errorMsg: string): Promise<{
+        output: undefined;
+        runner: TaskRunner;
+    }>;
+}
+export declare class TaskExecutionError extends Error {
+    runner: TaskRunner;
+    errorTask: ExecutionTask | null;
+    constructor(message: string, runner: TaskRunner, errorTask: ExecutionTask | null, options?: {
+        cause?: unknown;
+    });
+}
+export {};

package/dist/types/tree.d.ts

@@ -1 +1,4 @@
-export { truncateText, trimAttributes, descriptionOfTree, } from '@midscene/shared/extractor';
+import type { BaseElement, ElementTreeNode } from '@midscene/shared/types';
+import { trimAttributes, truncateText } from '@midscene/shared/extractor';
+export { trimAttributes, truncateText };
+export declare function descriptionOfTree<ElementType extends BaseElement = BaseElement>(tree: ElementTreeNode<ElementType>, truncateTextLength?: number, filterNonTextContent?: boolean, visibleOnly?: boolean): string;