@midscene/core 0.26.5-beta-20250814080504.0 → 0.26.5-beta-20250814125155.0

package/dist/types/ai-model.d.ts

@@ -1,596 +0,0 @@
-import { actionParser } from '@ui-tars/action-parser';
-import { BaseElement } from '@midscene/shared/types';
-import { ChatCompletionMessageParam } from 'openai/resources/index';
-import type { ChatCompletionSystemMessageParam } from 'openai/resources/index';
-import type { ChatCompletionUserMessageParam } from 'openai/resources/index';
-import { ElementTreeNode } from '@midscene/shared/types';
-import type { NodeType } from '@midscene/shared/constants';
-import OpenAI from 'openai';
-import { Rect } from '@midscene/shared/types';
-import { Size } from '@midscene/shared/types';
-import type { vlLocateMode } from '@midscene/shared/env';
-
-export declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number): Rect;
-
-export declare enum AIActionType {
-    ASSERT = 0,
-    INSPECT_ELEMENT = 1,
-    EXTRACT_DATA = 2,
-    PLAN = 3,
-    DESCRIBE_ELEMENT = 4
-}
-
-export declare type AIArgs = [
-ChatCompletionSystemMessageParam,
-...ChatCompletionUserMessageParam[]
-];
-
-export declare function AiAssert<ElementType extends BaseElement = BaseElement>(options: {
-    assertion: TUserPrompt;
-    context: UIContext<ElementType>;
-}): Promise<{
-    content: AIAssertionResponse;
-    usage: AIUsageInfo | undefined;
-}>;
-
-declare interface AIAssertionResponse {
-    pass: boolean;
-    thought: string;
-}
-
-declare interface AIDataExtractionResponse<DataDemand> {
-    data: DataDemand;
-    errors?: string[];
-    thought?: string;
-}
-
-declare interface AIElementCoordinatesResponse {
-    bbox: [number, number, number, number];
-    isOrderSensitive?: boolean;
-    errors?: string[];
-}
-
-declare interface AIElementLocatorResponse {
-    elements: {
-        id: string;
-        reason?: string;
-        text?: string;
-        xpaths?: string[];
-    }[];
-    bbox?: [number, number, number, number];
-    isOrderSensitive?: boolean;
-    errors?: string[];
-}
-
-declare type AIElementResponse =
-| AIElementLocatorResponse
-| AIElementCoordinatesResponse;
-
-export declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseElement>(options: {
-    dataQuery: string | Record<string, string>;
-    multimodalPrompt?: TMultimodalPrompt;
-    context: UIContext<ElementType>;
-    extractOption?: InsightExtractOption;
-}): Promise<{
-    parseResult: AIDataExtractionResponse<T>;
-    elementById: (idOrIndexId: string) => ElementType;
-    usage: AIUsageInfo | undefined;
-}>;
-
-export declare function AiLocateElement<ElementType extends BaseElement = BaseElement>(options: {
-    context: UIContext<ElementType>;
-    targetElementDescription: TUserPrompt;
-    referenceImage?: ReferenceImage;
-    callAI?: typeof callAiFn<AIElementResponse | [number, number]>;
-    searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;
-}): Promise<{
-    parseResult: AIElementLocatorResponse;
-    rect?: Rect;
-    rawResponse: string;
-    elementById: ElementById;
-    usage?: AIUsageInfo;
-    isOrderSensitive?: boolean;
-}>;
-
-export declare function AiLocateSection(options: {
-    context: UIContext<BaseElement>;
-    sectionDescription: TUserPrompt;
-    callAI?: typeof callAiFn<AISectionLocatorResponse>;
-}): Promise<{
-    rect?: Rect;
-    imageBase64?: string;
-    error?: string;
-    rawResponse: string;
-    usage?: AIUsageInfo;
-}>;
-
-declare interface AISectionLocatorResponse {
-    bbox: [number, number, number, number];
-    references_bbox?: [number, number, number, number][];
-    error?: string;
-}
-
-declare type AIUsageInfo = Record<string, any> & {
-    prompt_tokens: number | undefined;
-    completion_tokens: number | undefined;
-    total_tokens: number | undefined;
-    time_cost: number | undefined;
-};
-
-export declare function callAi(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, responseFormat?: OpenAI.ChatCompletionCreateParams['response_format'] | OpenAI.ResponseFormatJSONObject, options?: {
-    stream?: boolean;
-    onChunk?: StreamingCallback;
-}): Promise<{
-    content: string;
-    usage?: AIUsageInfo;
-    isStreamed: boolean;
-}>;
-
-export declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
-    content: T;
-    usage?: AIUsageInfo;
-}>;
-
-export declare function callAiFnWithStringResponse<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
-    content: string;
-    usage?: AIUsageInfo;
-}>;
-
-export declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType): Promise<{
-    content: T;
-    usage?: AIUsageInfo;
-}>;
-
-export { ChatCompletionMessageParam }
-
-declare interface ChromeRecordedEvent {
-    type: string;
-    timestamp: number;
-    url?: string;
-    title?: string;
-    elementDescription?: string;
-    value?: string;
-    pageInfo?: any;
-    elementRect?: any;
-    screenshotBefore?: string;
-    screenshotAfter?: string;
-    screenshotWithBox?: string;
-}
-
-declare interface CodeGenerationChunk {
-    /** The incremental content chunk */
-    content: string;
-    /** The reasoning content */
-    reasoning_content: string;
-    /** The accumulated content so far */
-    accumulated: string;
-    /** Whether this is the final chunk */
-    isComplete: boolean;
-    /** Token usage information if available */
-    usage?: AIUsageInfo;
-}
-
-export declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>, opt?: {
-    truncateTextLength?: number;
-    filterNonTextContent?: boolean;
-    domIncluded?: boolean | 'visible-only';
-    visibleOnly?: boolean;
-}): Promise<{
-    description: string;
-    elementById(idOrIndexId: string): ElementType;
-    elementByPosition(position: {
-        x: number;
-        y: number;
-    }, size: {
-        width: number;
-        height: number;
-    }): BaseElement | undefined;
-    insertElementByPosition(position: {
-        x: number;
-        y: number;
-    }): ElementType;
-    size: {
-        width: number;
-        height: number;
-    };
-}>;
-
-declare interface DetailedLocateParam extends LocateOption {
-    prompt: TUserPrompt;
-    referenceImage?: ReferenceImage;
-}
-
-declare interface DeviceAction<ParamType = any> {
-    name: string;
-    description?: string;
-    paramSchema?: string;
-    paramDescription?: string;
-    location?: 'required' | 'optional' | false;
-    whatToLocate?: string; // what to locate if location is required or optional
-    call: (context: ExecutorContext, param: ParamType) => Promise<void> | void;
-}
-
-declare type ElementById = (id: string) => BaseElement | null;
-
-export declare function elementByPositionWithElementInfo(treeRoot: ElementTreeNode<BaseElement>, position: {
-    x: number;
-    y: number;
-}, options?: {
-    requireStrictDistance?: boolean;
-    filterPositionElements?: boolean;
-}): BaseElement | undefined;
-
-declare interface ExecutionRecorderItem {
-    type: 'screenshot';
-    ts: number;
-    screenshot?: string;
-    timing?: string;
-}
-
-declare type ExecutionTask<
-E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<
-any,
-any,
-any
->,
-> = E &
-ExecutionTaskReturn<
-E extends ExecutionTaskApply<any, any, infer TaskOutput, any>
-? TaskOutput
-: unknown,
-E extends ExecutionTaskApply<any, any, any, infer TaskLog>
-? TaskLog
-: unknown
-> & {
-    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';
-    error?: Error;
-    errorMessage?: string;
-    errorStack?: string;
-    timing?: {
-        start: number;
-        end?: number;
-        cost?: number;
-    };
-    usage?: AIUsageInfo;
-};
-
-declare interface ExecutionTaskApply<
-Type extends ExecutionTaskType = any,
-TaskParam = any,
-TaskOutput = any,
-TaskLog = any,
-> {
-    type: Type;
-    subType?: string;
-    param?: TaskParam;
-    thought?: string;
-    locate?: PlanningLocateParam | null;
-    pageContext?: UIContext;
-    executor: (
-    param: TaskParam,
-    context: ExecutorContext,
-    ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>
-    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>
-    | undefined
-    | void;
-}
-
-declare interface ExecutionTaskHitBy {
-    from: string;
-    context: Record<string, any>;
-}
-
-declare interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {
-    output?: TaskOutput;
-    log?: TaskLog;
-    recorder?: ExecutionRecorderItem[];
-    hitBy?: ExecutionTaskHitBy;
-}
-
-declare type ExecutionTaskType =
-| 'Planning'
-| 'Insight'
-| 'Action'
-| 'Assertion'
-| 'Log';
-
-declare interface ExecutorContext {
-    task: ExecutionTask;
-    element?: LocateResultElement | null;
-}
-
-/**
- * Generates Playwright test code from recorded events
- */
-export declare const generatePlaywrightTest: (events: ChromeRecordedEvent[], options?: PlaywrightGenerationOptions) => Promise<string>;
-
-/**
- * Generates Playwright test code from recorded events with streaming support
- */
-export declare const generatePlaywrightTestStream: (events: ChromeRecordedEvent[], options?: PlaywrightGenerationOptions & StreamingCodeGenerationOptions) => Promise<StreamingAIResponse>;
-
-/**
- * Generates YAML test configuration from recorded events using AI
- */
-export declare const generateYamlTest: (events: ChromeRecordedEvent[], options?: YamlGenerationOptions) => Promise<string>;
-
-/**
- * Generates YAML test configuration from recorded events using AI with streaming support
- */
-export declare const generateYamlTestStream: (events: ChromeRecordedEvent[], options?: YamlGenerationOptions & StreamingCodeGenerationOptions) => Promise<StreamingAIResponse>;
-
-declare interface InsightExtractOption {
-    domIncluded?: boolean | 'visible-only';
-    screenshotIncluded?: boolean;
-    returnThought?: boolean;
-}
-
-declare interface LocateOption {
-    deepThink?: boolean; // only available in vl model
-    cacheable?: boolean; // user can set this param to false to disable the cache for a single agent api
-    xpath?: string; // only available in web
-    pageContext?: UIContext<BaseElement>;
-}
-
-declare type LocateResultElement = {
-    id: string;
-    indexId?: number;
-    center: [number, number];
-    rect: Rect;
-    xpaths: string[];
-    attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    isOrderSensitive?: boolean;
-};
-
-declare type MidsceneYamlFlowItem =
-| MidsceneYamlFlowItemAIAction
-| MidsceneYamlFlowItemAIAssert
-| MidsceneYamlFlowItemAIQuery
-| MidsceneYamlFlowItemAIWaitFor
-| MidsceneYamlFlowItemAITap
-| MidsceneYamlFlowItemAIRightClick
-| MidsceneYamlFlowItemAIHover
-| MidsceneYamlFlowItemAIInput
-| MidsceneYamlFlowItemAIKeyboardPress
-| MidsceneYamlFlowItemAIScroll
-| MidsceneYamlFlowItemSleep
-| MidsceneYamlFlowItemLogScreenshot;
-
-declare interface MidsceneYamlFlowItemAIAction {
-    ai?: string; // this is the shortcut for aiAction
-    aiAction?: string;
-    aiActionProgressTips?: string[];
-    cacheable?: boolean;
-}
-
-declare interface MidsceneYamlFlowItemAIAssert {
-    aiAssert: string;
-    errorMessage?: string;
-}
-
-declare interface MidsceneYamlFlowItemAIHover extends LocateOption {
-    aiHover: TUserPrompt;
-}
-
-declare interface MidsceneYamlFlowItemAIInput extends LocateOption {
-    aiInput: string; // value to input
-    locate: TUserPrompt; // where to input
-}
-
-declare interface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {
-    aiKeyboardPress: string;
-    locate?: TUserPrompt; // where to press, optional
-}
-
-declare interface MidsceneYamlFlowItemAIQuery extends InsightExtractOption {
-    aiQuery: string;
-    name?: string;
-}
-
-declare interface MidsceneYamlFlowItemAIRightClick extends LocateOption {
-    aiRightClick: TUserPrompt;
-}
-
-declare interface MidsceneYamlFlowItemAIScroll
-extends LocateOption,
-ScrollParam {
-    aiScroll: null;
-    locate?: TUserPrompt; // which area to scroll, optional
-}
-
-declare interface MidsceneYamlFlowItemAITap extends LocateOption {
-    aiTap: TUserPrompt;
-}
-
-declare interface MidsceneYamlFlowItemAIWaitFor {
-    aiWaitFor: string;
-    timeout?: number;
-}
-
-declare interface MidsceneYamlFlowItemLogScreenshot {
-    logScreenshot?: string; // optional, the title of the screenshot
-    content?: string;
-}
-
-declare interface MidsceneYamlFlowItemSleep {
-    sleep: number;
-}
-
-declare type PageType =
-| 'puppeteer'
-| 'playwright'
-| 'static'
-| 'chrome-extension-proxy'
-| 'android';
-
-export declare function plan(userInstruction: string, opts: {
-    context: UIContext;
-    pageType: PageType;
-    actionSpace: DeviceAction[];
-    callAI?: typeof callAiFn<PlanningAIResponse>;
-    log?: string;
-    actionContext?: string;
-}): Promise<PlanningAIResponse>;
-
-declare interface PlanningAction<ParamType = any> {
-    thought?: string;
-    type:
-    | 'Locate'
-    | 'Tap'
-    | 'RightClick'
-    | 'Hover'
-    | 'Drag'
-    | 'Input'
-    | 'KeyboardPress'
-    | 'Scroll'
-    | 'Error'
-    | 'Assert'
-    | 'AssertWithoutThrow'
-    | 'Sleep'
-    | 'Finished'
-    | 'AndroidBackButton'
-    | 'AndroidHomeButton'
-    | 'AndroidRecentAppsButton'
-    | 'AndroidLongPress'
-    | 'AndroidPull';
-    param: ParamType;
-    locate?: PlanningLocateParam | null;
-}
-
-declare interface PlanningAIResponse {
-    action?: PlanningAction; // this is the qwen mode
-    actions?: PlanningAction[];
-    more_actions_needed_by_instruction: boolean;
-    log: string;
-    sleep?: number;
-    error?: string;
-    usage?: AIUsageInfo;
-    rawResponse?: string;
-    yamlFlow?: MidsceneYamlFlowItem[];
-    yamlString?: string;
-}
-
-/**
- * planning
- *
- */
-
-declare interface PlanningLocateParam extends DetailedLocateParam {
-    id?: string;
-    bbox?: [number, number, number, number];
-}
-
-declare interface PlaywrightGenerationOptions {
-    testName?: string;
-    includeScreenshots?: boolean;
-    includeTimestamps?: boolean;
-    maxScreenshots?: number;
-    description?: string;
-    viewportSize?: {
-        width: number;
-        height: number;
-    };
-    waitForNetworkIdle?: boolean;
-    waitForNetworkIdleTimeout?: number;
-}
-
-declare interface ReferenceImage {
-    base64: string;
-    rect?: Rect;
-}
-
-export declare function resizeImageForUiTars(imageBase64: string, size: Size): Promise<string>;
-
-declare interface ScrollParam {
-    direction: 'down' | 'up' | 'right' | 'left';
-    scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
-    distance?: null | number; // distance in px
-}
-
-declare interface StreamingAIResponse {
-    /** The final accumulated content */
-    content: string;
-    /** Token usage information */
-    usage?: AIUsageInfo;
-    /** Whether the response was streamed */
-    isStreamed: boolean;
-}
-
-declare type StreamingCallback = (chunk: CodeGenerationChunk) => void;
-
-declare interface StreamingCodeGenerationOptions {
-    /** Whether to enable streaming output */
-    stream?: boolean;
-    /** Callback function to handle streaming chunks */
-    onChunk?: StreamingCallback;
-    /** Callback function to handle streaming completion */
-    onComplete?: (finalCode: string) => void;
-    /** Callback function to handle streaming errors */
-    onError?: (error: Error) => void;
-}
-
-export declare function systemPromptToLocateElement(vlMode: ReturnType<typeof vlLocateMode>): string;
-
-declare type TMultimodalPrompt = {
-    /**
-     * Support use image to inspect elements.
-     * The "images" field is an object that uses image name as key and image url as value.
-     * The image url can be a local path, a http link , or a base64 string.
-     */
-    images?: {
-        name: string;
-        url: string;
-    }[];
-    /**
-     * By default, the image url in the "images" filed starts with `https://` or `http://` will be directly sent to the LLM.
-     * In case the images are not accessible to the LLM (One common case is that image url is internal network only.), you can enable this option.
-     * Then image will be download and convert to base64 format.
-     */
-    convertHttpImage2Base64?: boolean;
-};
-
-declare type TUserPrompt =
-| string
-| ({
-    prompt: string;
-} & Partial<TMultimodalPrompt>);
-
-/**
- * context
- */
-
-declare abstract class UIContext<ElementType extends BaseElement = BaseElement> {
-    abstract screenshotBase64: string;
-
-    abstract tree: ElementTreeNode<ElementType>;
-
-    abstract size: Size;
-}
-
-export declare function vlmPlanning(options: {
-    userInstruction: string;
-    conversationHistory: ChatCompletionMessageParam[];
-    size: {
-        width: number;
-        height: number;
-    };
-}): Promise<{
-    actions: PlanningAction<any>[];
-    actionsFromModel: ReturnType<typeof actionParser>['parsed'];
-    action_summary: string;
-    yamlFlow?: MidsceneYamlFlowItem[];
-    usage?: AIUsageInfo;
-    rawResponse?: string;
-}>;
-
-declare interface YamlGenerationOptions {
-    testName?: string;
-    includeTimestamps?: boolean;
-    maxScreenshots?: number;
-    description?: string;
-}
-
-export { }