@midscene/core 0.30.10 → 1.0.0

package/dist/types/agent/ui-utils.d.ts

@@ -1,7 +1,14 @@
-import type { AndroidPullParam, DetailedLocateParam, ExecutionTask, ScrollParam } from '../types';
+import type { DetailedLocateParam, ExecutionTask, PullParam, ScrollParam } from '../types';
 export declare function typeStr(task: ExecutionTask): any;
 export declare function locateParamStr(locate?: DetailedLocateParam | string): string;
 export declare function scrollParamStr(scrollParam?: ScrollParam): string;
-export declare function pullParamStr(pullParam?: AndroidPullParam): string;
+export declare function pullParamStr(pullParam?: PullParam): string;
+export declare function extractInsightParam(taskParam: any): {
+    content: string;
+    images?: Array<{
+        name: string;
+        url: string;
+    }>;
+};
 export declare function taskTitleStr(type: 'Tap' | 'Hover' | 'Input' | 'RightClick' | 'KeyboardPress' | 'Scroll' | 'Action' | 'Query' | 'Assert' | 'WaitFor' | 'Locate' | 'Boolean' | 'Number' | 'String', prompt: string): string;
 export declare function paramStr(task: ExecutionTask): string;

package/dist/types/agent/utils.d.ts

@@ -1,7 +1,7 @@
-import type { TMultimodalPrompt, TUserPrompt } from '../ai-model/common';
+import type { TMultimodalPrompt, TUserPrompt } from '../common';
 import type { AbstractInterface } from '../device';
-import type { BaseElement, ElementCacheFeature, ElementTreeNode, ExecutionDump, ExecutorContext, LocateResultElement, PlanningLocateParam, UIContext } from '../types';
-import type { TaskExecutor } from './tasks';
+import type { ElementCacheFeature, LocateResultElement, PlanningLocateParam, UIContext } from '../types';
+import type { TaskCache } from './task-cache';
 export declare function commonContextParser(interfaceInstance: AbstractInterface, _opt: {
     uploadServerUrl?: string;
 }): Promise<UIContext>;
@@ -13,38 +13,12 @@ export declare function printReportMsg(filepath: string): void;
  */
 export declare function getCurrentExecutionFile(trace?: string): string | false;
 export declare function generateCacheId(fileName?: string): string;
-export declare function matchElementFromPlan(planLocateParam: PlanningLocateParam, tree: ElementTreeNode<BaseElement>): any;
-export declare function matchElementFromCache(taskExecutor: TaskExecutor, cacheEntry: ElementCacheFeature | undefined, cachePrompt: TUserPrompt, cacheable: boolean | undefined): Promise<LocateResultElement | undefined>;
-export declare function trimContextByViewport(execution: ExecutionDump): {
-    tasks: {
-        type: any;
-        subType?: string;
-        param?: any;
-        thought?: string;
-        locate?: PlanningLocateParam | null;
-        uiContext?: UIContext;
-        executor: (param: any, context: ExecutorContext) => void | Promise<void | import("../types").ExecutionTaskReturn<any, any> | undefined> | undefined;
-        output?: any;
-        log?: any;
-        recorder?: import("../types").ExecutionRecorderItem[];
-        hitBy?: import("../types").ExecutionTaskHitBy;
-        status: "pending" | "running" | "finished" | "failed" | "cancelled";
-        error?: Error;
-        errorMessage?: string;
-        errorStack?: string;
-        timing?: {
-            start: number;
-            end?: number;
-            cost?: number;
-        };
-        usage?: import("../types").AIUsageInfo;
-        searchAreaUsage?: import("../types").AIUsageInfo;
-    }[];
-    name: string;
-    description?: string;
-    aiActionContext?: string;
-    logTime: number;
-};
+export declare function ifPlanLocateParamIsBbox(planLocateParam: PlanningLocateParam): boolean;
+export declare function matchElementFromPlan(planLocateParam: PlanningLocateParam): LocateResultElement | undefined;
+export declare function matchElementFromCache(context: {
+    taskCache?: TaskCache;
+    interfaceInstance: AbstractInterface;
+}, cacheEntry: ElementCacheFeature | undefined, cachePrompt: TUserPrompt, cacheable: boolean | undefined): Promise<LocateResultElement | undefined>;
 export declare const getMidsceneVersion: () => string;
 export declare const parsePrompt: (prompt: TUserPrompt) => {
     textPrompt: string;

package/dist/types/ai-model/conversation-history.d.ts

@@ -1,18 +1,22 @@
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
 export interface ConversationHistoryOptions {
-    maxUserImageMessages?: number;
     initialMessages?: ChatCompletionMessageParam[];
 }
 export declare class ConversationHistory {
-    private readonly maxUserImageMessages;
     private readonly messages;
+    pendingFeedbackMessage: string;
     constructor(options?: ConversationHistoryOptions);
+    resetPendingFeedbackMessageIfExists(): void;
     append(message: ChatCompletionMessageParam): void;
     seed(messages: ChatCompletionMessageParam[]): void;
     reset(): void;
-    snapshot(): ChatCompletionMessageParam[];
+    /**
+     * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.
+     * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.
+     * @returns The snapshot of the conversation history.
+     */
+    snapshot(maxImages?: number): ChatCompletionMessageParam[];
     get length(): number;
     [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam>;
     toJSON(): ChatCompletionMessageParam[];
-    private pruneOldestUserMessageIfNecessary;
 }

package/dist/types/ai-model/index.d.ts

@@ -1,13 +1,13 @@
 export { callAIWithStringResponse, callAIWithObjectResponse, callAI, } from './service-caller/index';
 export { systemPromptToLocateElement } from './prompt/llm-locator';
-export { describeUserPage, elementByPositionWithElementInfo, } from './prompt/util';
+export { describeUserPage } from './prompt/util';
 export { generatePlaywrightTest, generatePlaywrightTestStream, } from './prompt/playwright-generator';
 export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generator';
 export type { ChatCompletionMessageParam } from 'openai/resources/index';
-export { AiLocateElement, AiExtractElementInfo, AiLocateSection, } from './inspect';
+export { AiLocateElement, AiExtractElementInfo, AiLocateSection, AiJudgeOrderSensitive, } from './inspect';
 export { plan } from './llm-planning';
-export { adaptBboxToRect } from './common';
+export { adaptBboxToRect } from '../common';
 export { uiTarsPlanning, resizeImageForUiTars } from './ui-tars-planning';
 export { ConversationHistory, type ConversationHistoryOptions, } from './conversation-history';
-export { AIActionType, type AIArgs } from './common';
-export { getMidsceneLocationSchema, type MidsceneLocationResultType, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, findAllMidsceneLocatorField, dumpActionParam, loadActionParam, parseActionParam, } from './common';
+export { AIActionType, type AIArgs } from '../common';
+export { getMidsceneLocationSchema, type MidsceneLocationResultType, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, findAllMidsceneLocatorField, dumpActionParam, loadActionParam, parseActionParam, } from '../common';

package/dist/types/ai-model/inspect.d.ts

@@ -1,29 +1,31 @@
-import type { AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, UIContext } from '../types';
+import type { AIDataExtractionResponse, AIElementResponse, AIUsageInfo, Rect, ReferenceImage, ServiceExtractOption, UIContext } from '../types';
 import type { IModelConfig } from '@midscene/shared/env';
+import type { LocateResultElement } from '@midscene/shared/types';
 import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
-import type { TMultimodalPrompt, TUserPrompt } from './common';
+import type { TMultimodalPrompt, TUserPrompt } from '../common';
 import { callAIWithObjectResponse } from './service-caller/index';
 export type AIArgs = [
     ChatCompletionSystemMessageParam,
     ...ChatCompletionUserMessageParam[]
 ];
-export declare function AiLocateElement<ElementType extends BaseElement = BaseElement>(options: {
-    context: UIContext<ElementType>;
+export declare function AiLocateElement(options: {
+    context: UIContext;
     targetElementDescription: TUserPrompt;
     referenceImage?: ReferenceImage;
     callAIFn: typeof callAIWithObjectResponse<AIElementResponse | [number, number]>;
     searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;
     modelConfig: IModelConfig;
 }): Promise<{
-    parseResult: AIElementLocatorResponse;
+    parseResult: {
+        elements: LocateResultElement[];
+        errors?: string[];
+    };
     rect?: Rect;
     rawResponse: string;
-    elementById: ElementById;
     usage?: AIUsageInfo;
-    isOrderSensitive?: boolean;
 }>;
 export declare function AiLocateSection(options: {
-    context: UIContext<BaseElement>;
+    context: UIContext;
     sectionDescription: TUserPrompt;
     modelConfig: IModelConfig;
 }): Promise<{
@@ -33,14 +35,20 @@ export declare function AiLocateSection(options: {
     rawResponse: string;
     usage?: AIUsageInfo;
 }>;
-export declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseElement>(options: {
+export declare function AiExtractElementInfo<T>(options: {
     dataQuery: string | Record<string, string>;
     multimodalPrompt?: TMultimodalPrompt;
-    context: UIContext<ElementType>;
-    extractOption?: InsightExtractOption;
+    context: UIContext;
+    pageDescription?: string;
+    extractOption?: ServiceExtractOption;
     modelConfig: IModelConfig;
 }): Promise<{
     parseResult: AIDataExtractionResponse<T>;
-    elementById: (idOrIndexId: string) => ElementType;
     usage: AIUsageInfo | undefined;
 }>;
+export declare function AiJudgeOrderSensitive(description: string, callAIFn: typeof callAIWithObjectResponse<{
+    isOrderSensitive: boolean;
+}>, modelConfig: IModelConfig): Promise<{
+    isOrderSensitive: boolean;
+    usage?: AIUsageInfo;
+}>;

package/dist/types/ai-model/llm-planning.d.ts

@@ -7,5 +7,7 @@ export declare function plan(userInstruction: string, opts: {
     actionSpace: DeviceAction<any>[];
     actionContext?: string;
     modelConfig: IModelConfig;
-    conversationHistory?: ConversationHistory;
+    conversationHistory: ConversationHistory;
+    includeBbox: boolean;
+    imagesIncludeCount?: number;
 }): Promise<PlanningAIResponse>;

package/dist/types/ai-model/prompt/llm-locator.d.ts

@@ -1,8 +1,3 @@
 import type { TVlModeTypes } from '@midscene/shared/env';
-import type { ResponseFormatJSONSchema } from 'openai/resources/index';
 export declare function systemPromptToLocateElement(vlMode: TVlModeTypes | undefined): string;
-export declare const locatorSchema: ResponseFormatJSONSchema;
-export declare const findElementPrompt: ({ pageDescription, targetElementDescription, }: {
-    pageDescription: string;
-    targetElementDescription: string;
-}) => string;
+export declare const findElementPrompt: (targetElementDescription: string) => string;

package/dist/types/ai-model/prompt/llm-planning.d.ts

@@ -1,9 +1,8 @@
 import type { DeviceAction } from '../../types';
 import type { TVlModeTypes } from '@midscene/shared/env';
-import type { ResponseFormatJSONSchema } from 'openai/resources/index';
 export declare const descriptionForAction: (action: DeviceAction<any>, locatorSchemaTypeDescription: string) => string;
-export declare function systemPromptToTaskPlanning({ actionSpace, vlMode, }: {
+export declare function systemPromptToTaskPlanning({ actionSpace, vlMode, includeBbox, }: {
     actionSpace: DeviceAction<any>[];
     vlMode: TVlModeTypes | undefined;
+    includeBbox: boolean;
 }): Promise<string>;
-export declare const planSchema: ResponseFormatJSONSchema;

package/dist/types/ai-model/prompt/llm-section-locator.d.ts

@@ -1,5 +1,3 @@
 import type { TVlModeTypes } from '@midscene/shared/env';
 export declare function systemPromptToLocateSection(vlMode: TVlModeTypes | undefined): string;
-export declare const sectionLocatorInstruction: ({ sectionDescription, }: {
-    sectionDescription: string;
-}) => string;
+export declare const sectionLocatorInstruction: (sectionDescription: string) => string;

package/dist/types/ai-model/prompt/order-sensitive-judge.d.ts

@@ -0,0 +1,2 @@
+export declare function systemPromptToJudgeOrderSensitive(): string;
+export declare const orderSensitiveJudgePrompt: (description: string) => string;

package/dist/types/ai-model/prompt/util.d.ts

@@ -1,17 +1,9 @@
-import type { BaseElement, ElementTreeNode, Size, UIContext } from '../../types';
-import type { TVlModeTypes } from '@midscene/shared/env';
+import type { BaseElement, Size, UIContext } from '../../types';
 export declare function describeSize(size: Size): string;
 export declare function describeElement(elements: (Pick<BaseElement, 'rect' | 'content'> & {
     id: string;
 })[]): string;
 export declare const distanceThreshold = 16;
-export declare function elementByPositionWithElementInfo(treeRoot: ElementTreeNode<BaseElement>, position: {
-    x: number;
-    y: number;
-}, options?: {
-    requireStrictDistance?: boolean;
-    filterPositionElements?: boolean;
-}): BaseElement | undefined;
 export declare function distance(point1: {
     x: number;
     y: number;
@@ -20,28 +12,4 @@ export declare function distance(point1: {
     y: number;
 }): number;
 export declare const samplePageDescription = "\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n";
-export declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>, opt: {
-    truncateTextLength?: number;
-    filterNonTextContent?: boolean;
-    domIncluded?: boolean | 'visible-only';
-    visibleOnly?: boolean;
-    vlMode: TVlModeTypes | undefined;
-}): Promise<{
-    description: string;
-    elementById(idOrIndexId: string): ElementType;
-    elementByPosition(position: {
-        x: number;
-        y: number;
-    }, size: {
-        width: number;
-        height: number;
-    }): BaseElement | undefined;
-    insertElementByPosition(position: {
-        x: number;
-        y: number;
-    }): ElementType;
-    size: {
-        width: number;
-        height: number;
-    };
-}>;
+export declare function describeUserPage(context: UIContext): Promise<string>;

package/dist/types/ai-model/service-caller/index.d.ts

@@ -1,9 +1,8 @@
 import { type AIUsageInfo } from '../../types';
 import type { StreamingCallback } from '../../types';
 import { type IModelConfig, type TVlModeTypes } from '@midscene/shared/env';
-import OpenAI from 'openai';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
-import { AIActionType, type AIArgs } from '../common';
+import type { AIActionType, AIArgs } from '../../common';
 export declare function callAI(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelConfig: IModelConfig, options?: {
     stream?: boolean;
     onChunk?: StreamingCallback;
@@ -12,9 +11,9 @@ export declare function callAI(messages: ChatCompletionMessageParam[], AIActionT
     usage?: AIUsageInfo;
     isStreamed: boolean;
 }>;
-export declare const getResponseFormat: (modelName: string, AIActionTypeValue: AIActionType) => OpenAI.ChatCompletionCreateParams["response_format"] | OpenAI.ResponseFormatJSONObject;
 export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelConfig: IModelConfig): Promise<{
     content: T;
+    contentString: string;
     usage?: AIUsageInfo;
 }>;
 export declare function callAIWithStringResponse(msgs: AIArgs, AIActionTypeValue: AIActionType, modelConfig: IModelConfig): Promise<{

package/dist/types/ai-model/ui-tars-planning.d.ts

@@ -1,11 +1,12 @@
 import type { PlanningAIResponse, Size, UIContext } from '../types';
 import { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';
 import type { ConversationHistory } from './conversation-history';
-type ActionType = 'click' | 'drag' | 'type' | 'hotkey' | 'finished' | 'scroll' | 'wait';
+type ActionType = 'click' | 'left_double' | 'right_single' | 'drag' | 'type' | 'hotkey' | 'finished' | 'scroll' | 'wait';
 export declare function uiTarsPlanning(userInstruction: string, options: {
     conversationHistory: ConversationHistory;
     context: UIContext;
     modelConfig: IModelConfig;
+    actionContext?: string;
 }): Promise<PlanningAIResponse>;
 interface BaseAction {
     action_type: ActionType;
@@ -32,6 +33,18 @@ interface WaitAction extends BaseAction {
         time: string;
     };
 }
+interface LeftDoubleAction extends BaseAction {
+    action_type: 'left_double';
+    action_inputs: {
+        start_box: string;
+    };
+}
+interface RightSingleAction extends BaseAction {
+    action_type: 'right_single';
+    action_inputs: {
+        start_box: string;
+    };
+}
 interface TypeAction extends BaseAction {
     action_type: 'type';
     action_inputs: {
@@ -54,6 +67,6 @@ interface FinishedAction extends BaseAction {
     action_type: 'finished';
     action_inputs: Record<string, never>;
 }
-export type Action = ClickAction | DragAction | TypeAction | HotkeyAction | ScrollAction | FinishedAction | WaitAction;
+export type Action = ClickAction | LeftDoubleAction | RightSingleAction | DragAction | TypeAction | HotkeyAction | ScrollAction | FinishedAction | WaitAction;
 export declare function resizeImageForUiTars(imageBase64: string, size: Size, uiTarsVersion: UITarsModelVersion | undefined): Promise<string>;
 export {};

package/dist/types/{ai-model/common.d.ts → common.d.ts}

@@ -1,6 +1,6 @@
-import type { BaseElement, DeviceAction, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from '../types';
+import type { BaseElement, DeviceAction, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from './types';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
-import type { PlanningLocateParam } from '../types';
+import type { PlanningLocateParam } from './types';
 import type { TVlModeTypes } from '@midscene/shared/env';
 import { z } from 'zod';
 export type AIArgs = ChatCompletionMessageParam[];
@@ -12,14 +12,14 @@ export declare enum AIActionType {
     DESCRIBE_ELEMENT = 4,
     TEXT = 5
 }
+type AdaptBboxInput = number[] | string[] | string | (number[] | string[])[];
 export declare function fillBboxParam(locate: PlanningLocateParam, width: number, height: number, rightLimit: number, bottomLimit: number, vlMode: TVlModeTypes | undefined): PlanningLocateParam;
-export declare function adaptQwenBbox(bbox: number[]): [number, number, number, number];
+export declare function adaptQwen2_5Bbox(bbox: number[]): [number, number, number, number];
 export declare function adaptDoubaoBbox(bbox: string[] | number[] | string, width: number, height: number): [number, number, number, number];
-export declare function adaptBbox(bbox: number[], width: number, height: number, rightLimit: number, bottomLimit: number, vlMode: TVlModeTypes | undefined): [number, number, number, number];
+export declare function adaptBbox(bbox: AdaptBboxInput, width: number, height: number, rightLimit: number, bottomLimit: number, vlMode: TVlModeTypes | undefined): [number, number, number, number];
 export declare function normalized01000(bbox: number[], width: number, height: number): [number, number, number, number];
 export declare function adaptGeminiBbox(bbox: number[], width: number, height: number): [number, number, number, number];
 export declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number, rightLimit?: number, bottomLimit?: number, vlMode?: TVlModeTypes | undefined): Rect;
-export declare function warnGPT4oSizeLimit(size: Size, modelName: string): void;
 export declare function mergeRects(rects: Rect[]): {
     left: number;
     top: number;
@@ -565,5 +565,5 @@ export declare const loadActionParam: (jsonObject: Record<string, any>, zodSchem
  * Locator fields are special business logic fields with complex validation requirements,
  * so they are intentionally excluded from Zod parsing and use existing validation logic.
  */
-export declare const parseActionParam: (rawParam: Record<string, any>, zodSchema: z.ZodType<any>) => Record<string, any>;
+export declare const parseActionParam: (rawParam: Record<string, any> | undefined, zodSchema?: z.ZodType<any>) => Record<string, any> | undefined;
 export {};

package/dist/types/device/device-options.d.ts

@@ -0,0 +1,57 @@
+import type { DeviceAction } from '../types';
+/**
+ * Android device input options
+ */
+export type AndroidDeviceInputOpt = {
+    /** Automatically dismiss the keyboard after input is completed */
+    autoDismissKeyboard?: boolean;
+    /** Strategy for dismissing the keyboard: 'esc-first' tries ESC before BACK, 'back-first' tries BACK before ESC */
+    keyboardDismissStrategy?: 'esc-first' | 'back-first';
+};
+/**
+ * Android device options
+ */
+export type AndroidDeviceOpt = {
+    /** Path to the ADB executable */
+    androidAdbPath?: string;
+    /** Remote ADB host address */
+    remoteAdbHost?: string;
+    /** Remote ADB port */
+    remoteAdbPort?: number;
+    /** Input method editor strategy: 'always-yadb' always uses yadb, 'yadb-for-non-ascii' uses yadb only for non-ASCII characters */
+    imeStrategy?: 'always-yadb' | 'yadb-for-non-ascii';
+    /** Display ID to use for this device */
+    displayId?: number;
+    /** Use physical display ID for screenshot operations */
+    usePhysicalDisplayIdForScreenshot?: boolean;
+    /** Use physical display ID when looking up display information */
+    usePhysicalDisplayIdForDisplayLookup?: boolean;
+    /** Custom device actions to register */
+    customActions?: DeviceAction<any>[];
+    /** Screenshot resize scale factor */
+    screenshotResizeScale?: number;
+    /** Always fetch screen info on each call; if false, cache the first result */
+    alwaysRefreshScreenInfo?: boolean;
+} & AndroidDeviceInputOpt;
+/**
+ * iOS device input options
+ */
+export type IOSDeviceInputOpt = {
+    /** Automatically dismiss the keyboard after input is completed */
+    autoDismissKeyboard?: boolean;
+};
+/**
+ * iOS device options
+ */
+export type IOSDeviceOpt = {
+    /** Device ID (UDID) to connect to */
+    deviceId?: string;
+    /** Custom device actions to register */
+    customActions?: DeviceAction<any>[];
+    /** WebDriverAgent port (default: 8100) */
+    wdaPort?: number;
+    /** WebDriverAgent host (default: 'localhost') */
+    wdaHost?: string;
+    /** Whether to use WebDriverAgent */
+    useWDA?: boolean;
+} & IOSDeviceInputOpt;