@midscene/core 0.26.2-beta-20250812091127.0 → 0.26.3-beta-20250813021342.0

package/dist/types/index.d.ts

@@ -1,53 +1,1299 @@
-import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractParam, h as InsightExtractOption, T as TMultimodalPrompt, A as AIUsageInfo, i as TUserPrompt, j as InsightAssertionResponse, k as AIDescribeElementResponse } from './types-8a6be57c.js';
-export { z as AIAssertionResponse, x as AIDataExtractionResponse, v as AIElementCoordinatesResponse, u as AIElementLocatorResponse, w as AIElementResponse, q as AIResponseFormat, y as AISectionLocatorResponse, t as AISingleElementResponse, r as AISingleElementResponseById, s as AISingleElementResponseByPosition, W as AgentAssertOpt, F as AgentDescribeElementAtPointResult, V as AgentWaitForOpt, aa as BaseAgentParserOpt, G as CallAIFn, aE as CodeGenerationChunk, a9 as Color, aG as DeviceAction, K as DumpMeta, Q as ElementById, H as EnsureObject, ad as ExecutionRecorderItem, av as ExecutionTaskAction, au as ExecutionTaskActionApply, ag as ExecutionTaskHitBy, at as ExecutionTaskInsightAssertion, as as ExecutionTaskInsightAssertionApply, ar as ExecutionTaskInsightAssertionParam, ak as ExecutionTaskInsightDumpLog, am as ExecutionTaskInsightLocate, al as ExecutionTaskInsightLocateApply, aj as ExecutionTaskInsightLocateOutput, ai as ExecutionTaskInsightLocateParam, aq as ExecutionTaskInsightQuery, ap as ExecutionTaskInsightQueryApply, ao as ExecutionTaskInsightQueryOutput, an as ExecutionTaskInsightQueryParam, ax as ExecutionTaskLog, aw as ExecutionTaskLogApply, az as ExecutionTaskPlanning, ay as ExecutionTaskPlanningApply, ah as ExecutionTaskReturn, ae as ExecutionTaskType, af as ExecutorContext, b2 as FreeFn, aA as GroupedActionDump, N as InsightDump, O as LiteUISection, p as LocateOption, J as LocateResultElement, C as LocateValidatorResult, B as LocatorValidatorOption, b5 as MidsceneYamlConfig, b6 as MidsceneYamlConfigOutput, o as MidsceneYamlConfigResult, m as MidsceneYamlFlowItem, aN as MidsceneYamlFlowItemAIAction, aS as MidsceneYamlFlowItemAIAsk, aO as MidsceneYamlFlowItemAIAssert, aT as MidsceneYamlFlowItemAIBoolean, aX as MidsceneYamlFlowItemAIHover, aY as MidsceneYamlFlowItemAIInput, aZ as MidsceneYamlFlowItemAIKeyboardPress, aU as MidsceneYamlFlowItemAILocate, aQ as MidsceneYamlFlowItemAINumber, aP as MidsceneYamlFlowItemAIQuery, n as MidsceneYamlFlowItemAIRightClick, a_ as MidsceneYamlFlowItemAIScroll, aR as MidsceneYamlFlowItemAIString, aW as MidsceneYamlFlowItemAITap, aV as MidsceneYamlFlowItemAIWaitFor, a$ as MidsceneYamlFlowItemEvaluateJavaScript, b1 as MidsceneYamlFlowItemLogScreenshot, b0 as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aL as MidsceneYamlScriptAndroidEnv, aM as MidsceneYamlScriptEnv, aJ as MidsceneYamlScriptEnvBase, aK as MidsceneYamlScriptWebEnv, l as MidsceneYamlTask, S as OnTaskStartTip, aB as PageType, P as PartialInsightDumpFromSDK, Z as PlanningAIResponse, Y as PlanningAction, a7 as PlanningActionParamAndroidLongPress, a8 as PlanningActionParamAndroidPull, a3 as PlanningActionParamAssert, a5 as PlanningActionParamError, $ as PlanningActionParamHover, a1 as PlanningActionParamInputOrKeyPress, a0 as PlanningActionParamRightClick, a2 as PlanningActionParamScroll, a4 as PlanningActionParamSleep, _ as PlanningActionParamTap, a6 as PlanningActionParamWaitFor, X as PlanningLocateParam, ac as PlaywrightParserOpt, ab as PuppeteerParserOpt, aH as ReferenceImage, R as ReportDumpWithAttributes, b4 as ScriptPlayerStatusValue, b3 as ScriptPlayerTaskStatus, aF as StreamingAIResponse, aD as StreamingCallback, aC as StreamingCodeGenerationOptions, aI as scrollParam } from './types-8a6be57c.js';
-import { c as callAiFn } from './llm-planning-4e0c16fe.js';
-export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-4e0c16fe.js';
-import { BaseElement, Rect } from '@midscene/shared/types';
-export { BaseElement, ElementTreeNode, Point, Rect, Size } from '@midscene/shared/types';
-export { getVersion } from './utils.js';
-export { MIDSCENE_MODEL_NAME, getAIConfig } from '@midscene/shared/env';
-import '@midscene/shared/constants';
-import 'openai/resources';
-
-declare class Executor {
-    name: string;
-    tasks: ExecutionTask[];
-    status: 'init' | 'pending' | 'running' | 'completed' | 'error';
-    onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
-    constructor(name: string, options?: ExecutionTaskProgressOptions & {
-        tasks?: ExecutionTaskApply[];
-    });
-    private markTaskAsPending;
-    append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void>;
-    flush(): Promise<{
-        output: any;
-        thought?: string;
-    } | undefined>;
-    isInErrorState(): boolean;
-    latestErrorTask(): ExecutionTask | null;
-    dump(): ExecutionDump;
-}
-
-interface LocateOpts {
-    context?: UIContext<BaseElement>;
-    callAI?: typeof callAiFn<AIElementResponse>;
-}
-declare class Insight<ElementType extends BaseElement = BaseElement, ContextType extends UIContext<ElementType> = UIContext<ElementType>> {
-    contextRetrieverFn: (action: InsightAction) => Promise<ContextType> | ContextType;
-    aiVendorFn: (...args: Array<any>) => Promise<any>;
-    onceDumpUpdatedFn?: DumpSubscriber;
-    taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
-    constructor(context: ContextType | ((action: InsightAction) => Promise<ContextType> | ContextType), opt?: InsightOptions);
-    locate(query: DetailedLocateParam, opt?: LocateOpts): Promise<LocateResult>;
-    extract<T>(dataDemand: InsightExtractParam, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<{
-        data: T;
-        thought?: string;
-        usage?: AIUsageInfo;
-    }>;
-    assert(assertion: TUserPrompt): Promise<InsightAssertionResponse>;
-    describe(target: Rect | [number, number], opt?: {
-        deepThink?: boolean;
-    }): Promise<Pick<AIDescribeElementResponse, 'description'>>;
-}
-
-export { AIDescribeElementResponse, AIUsageInfo, DetailedLocateParam, DumpSubscriber, ExecutionDump, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions, Executor, Insight, InsightAction, InsightAssertionResponse, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, TMultimodalPrompt, TUserPrompt, UIContext, Insight as default };
+import { BaseElement } from '@midscene/shared/types';
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+import type { ChatCompletionSystemMessageParam } from 'openai/resources/index';
+import type { ChatCompletionUserMessageParam } from 'openai/resources/index';
+import { ElementTreeNode } from '@midscene/shared/types';
+import { getAIConfig } from '@midscene/shared/env';
+import { MIDSCENE_MODEL_NAME } from '@midscene/shared/env';
+import type { NodeType } from '@midscene/shared/constants';
+import { Point } from '@midscene/shared/types';
+import { Rect } from '@midscene/shared/types';
+import { Size } from '@midscene/shared/types';
+
+export declare interface AgentAssertOpt {
+    keepRawResponse?: boolean;
+}
+
+export declare interface AgentDescribeElementAtPointResult {
+    prompt: string;
+    deepThink: boolean;
+    verifyResult?: LocateValidatorResult;
+}
+
+export declare interface AgentWaitForOpt {
+    checkIntervalMs?: number;
+    timeoutMs?: number;
+}
+
+declare enum AIActionType {
+    ASSERT = 0,
+    INSPECT_ELEMENT = 1,
+    EXTRACT_DATA = 2,
+    PLAN = 3,
+    DESCRIBE_ELEMENT = 4
+}
+
+declare enum AIActionType_2 {
+    ASSERT = 0,
+    INSPECT_ELEMENT = 1,
+    EXTRACT_DATA = 2,
+    PLAN = 3,
+    DESCRIBE_ELEMENT = 4,
+}
+
+declare type AIArgs = [
+ChatCompletionSystemMessageParam,
+...ChatCompletionUserMessageParam[]
+];
+
+declare type AIArgs_2 = [
+ChatCompletionSystemMessageParam,
+...ChatCompletionUserMessageParam[],
+];
+
+export declare function AiAssert<ElementType extends BaseElement = BaseElement>(options: {
+    assertion: TUserPrompt_2;
+    context: UIContext_2<ElementType>;
+}): Promise<{
+    content: AIAssertionResponse_2;
+    usage: AIUsageInfo_2 | undefined;
+}>;
+
+export declare interface AIAssertionResponse {
+    pass: boolean;
+    thought: string;
+}
+
+declare interface AIAssertionResponse_2 {
+    pass: boolean;
+    thought: string;
+}
+
+export declare interface AIDataExtractionResponse<DataDemand> {
+    data: DataDemand;
+    errors?: string[];
+    thought?: string;
+}
+
+export declare interface AIDescribeElementResponse {
+    description: string;
+    error?: string;
+}
+
+declare interface AIDescribeElementResponse_2 {
+    description: string;
+    error?: string;
+}
+
+export declare interface AIElementCoordinatesResponse {
+    bbox: [number, number, number, number];
+    isOrderSensitive?: boolean;
+    errors?: string[];
+}
+
+declare interface AIElementCoordinatesResponse_2 {
+    bbox: [number, number, number, number];
+    isOrderSensitive?: boolean;
+    errors?: string[];
+}
+
+export declare interface AIElementLocatorResponse {
+    elements: {
+        id: string;
+        reason?: string;
+        text?: string;
+        xpaths?: string[];
+    }[];
+    bbox?: [number, number, number, number];
+    isOrderSensitive?: boolean;
+    errors?: string[];
+}
+
+declare interface AIElementLocatorResponse_2 {
+    elements: {
+        id: string;
+        reason?: string;
+        text?: string;
+        xpaths?: string[];
+    }[];
+    bbox?: [number, number, number, number];
+    isOrderSensitive?: boolean;
+    errors?: string[];
+}
+
+export declare type AIElementResponse = AIElementLocatorResponse | AIElementCoordinatesResponse;
+
+declare type AIElementResponse_2 =
+| AIElementLocatorResponse_2
+| AIElementCoordinatesResponse_2;
+
+export declare function AiLocateElement<ElementType extends BaseElement = BaseElement>(options: {
+    context: UIContext_2<ElementType>;
+    targetElementDescription: TUserPrompt_2;
+    referenceImage?: ReferenceImage_2;
+    callAI?: typeof callAiFn<AIElementResponse_2 | [number, number]>;
+    searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;
+}): Promise<{
+    parseResult: AIElementLocatorResponse_2;
+    rect?: Rect;
+    rawResponse: string;
+    elementById: ElementById_2;
+    usage?: AIUsageInfo_2;
+    isOrderSensitive?: boolean;
+}>;
+
+declare function AiLocateSection(options: {
+    context: UIContext_2<BaseElement>;
+    sectionDescription: TUserPrompt_2;
+    callAI?: typeof callAiFn<AISectionLocatorResponse_2>;
+}): Promise<{
+    rect?: Rect;
+    imageBase64?: string;
+    error?: string;
+    rawResponse: string;
+    usage?: AIUsageInfo_2;
+}>;
+
+/**
+ * openai
+ *
+ */
+export declare enum AIResponseFormat {
+    JSON = "json_object",
+    TEXT = "text"
+}
+
+export declare interface AISectionLocatorResponse {
+    bbox: [number, number, number, number];
+    references_bbox?: [number, number, number, number][];
+    error?: string;
+}
+
+declare interface AISectionLocatorResponse_2 {
+    bbox: [number, number, number, number];
+    references_bbox?: [number, number, number, number][];
+    error?: string;
+}
+
+export declare type AISingleElementResponse = AISingleElementResponseById;
+
+export declare type AISingleElementResponseById = {
+    id: string;
+    reason?: string;
+    text?: string;
+    xpaths?: string[];
+};
+
+export declare type AISingleElementResponseByPosition = {
+    position?: {
+        x: number;
+        y: number;
+    };
+    bbox?: [number, number, number, number];
+    reason: string;
+    text: string;
+};
+
+export declare type AIUsageInfo = Record<string, any> & {
+    prompt_tokens: number | undefined;
+    completion_tokens: number | undefined;
+    total_tokens: number | undefined;
+    time_cost: number | undefined;
+};
+
+declare type AIUsageInfo_2 = Record<string, any> & {
+    prompt_tokens: number | undefined;
+    completion_tokens: number | undefined;
+    total_tokens: number | undefined;
+    time_cost: number | undefined;
+};
+
+export declare interface BaseAgentParserOpt {
+    selector?: string;
+    ignoreMarker?: boolean;
+}
+
+export { BaseElement }
+
+/**
+ * insight
+ */
+export declare type CallAIFn = <T>(messages: ChatCompletionMessageParam[]) => Promise<T>;
+
+declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
+    content: T;
+    usage?: AIUsageInfo_2;
+}>;
+
+/**
+ * insight
+ */
+
+declare type CallAIFn_2 = <T>(
+messages: ChatCompletionMessageParam[],
+) => Promise<T>;
+
+declare async function callAiFn_2<T>(
+msgs: AIArgs_2,
+AIActionTypeValue: AIActionType_2,
+): Promise<{ content: T; usage?: AIUsageInfo_2 }> {
+    const jsonObject = await callToGetJSONObject<T>(msgs, AIActionTypeValue);
+
+    return {
+        content: jsonObject.content,
+        usage: jsonObject.usage,
+    };
+}
+
+export declare interface CodeGenerationChunk {
+    /** The incremental content chunk */
+    content: string;
+    /** The reasoning content */
+    reasoning_content: string;
+    /** The accumulated content so far */
+    accumulated: string;
+    /** Whether this is the final chunk */
+    isComplete: boolean;
+    /** Token usage information if available */
+    usage?: AIUsageInfo;
+}
+
+/**
+ * misc
+ */
+export declare interface Color {
+    name: string;
+    hex: string;
+}
+
+export declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext_2<ElementType>, 'describer'>, opt?: {
+    truncateTextLength?: number;
+    filterNonTextContent?: boolean;
+    domIncluded?: boolean | 'visible-only';
+    visibleOnly?: boolean;
+}): Promise<{
+    description: string;
+    elementById(idOrIndexId: string): ElementType;
+    elementByPosition(position: {
+        x: number;
+        y: number;
+    }, size: {
+        width: number;
+        height: number;
+    }): BaseElement | undefined;
+    insertElementByPosition(position: {
+        x: number;
+        y: number;
+    }): ElementType;
+    size: {
+        width: number;
+        height: number;
+    };
+}>;
+
+export declare interface DetailedLocateParam extends LocateOption {
+    prompt: TUserPrompt;
+    referenceImage?: ReferenceImage;
+}
+
+declare interface DetailedLocateParam_2 extends LocateOption_2 {
+    prompt: TUserPrompt_2;
+    referenceImage?: ReferenceImage_2;
+}
+
+export declare interface DeviceAction<ParamType = any> {
+    name: string;
+    description?: string;
+    paramSchema?: string;
+    paramDescription?: string;
+    location?: 'required' | 'optional' | false;
+    whatToLocate?: string;
+    call: (param: ParamType) => Promise<void> | void;
+}
+
+declare interface DeviceAction_2<ParamType = any> {
+    name: string;
+    description?: string;
+    paramSchema?: string;
+    paramDescription?: string;
+    location?: 'required' | 'optional' | false;
+    whatToLocate?: string; // what to locate if location is required or optional
+    call: (param: ParamType) => Promise<void> | void;
+}
+
+export declare interface DumpMeta {
+    sdkVersion: string;
+    logTime: number;
+    model_name: string;
+    model_description?: string;
+}
+
+declare interface DumpMeta_2 {
+    sdkVersion: string;
+    logTime: number;
+    model_name: string;
+    model_description?: string;
+}
+
+export declare type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;
+
+declare type DumpSubscriber_2 = (dump: InsightDump_2) => Promise<void> | void;
+
+export declare type ElementById = (id: string) => BaseElement | null;
+
+declare type ElementById_2 = (id: string) => BaseElement | null;
+
+export { ElementTreeNode }
+
+export declare type EnsureObject<T> = {
+    [K in keyof T]: any;
+};
+
+export declare interface ExecutionDump extends DumpMeta {
+    name: string;
+    description?: string;
+    tasks: ExecutionTask[];
+}
+
+declare interface ExecutionDump_2 extends DumpMeta_2 {
+    name: string;
+    description?: string;
+    tasks: ExecutionTask_2[];
+}
+
+export declare interface ExecutionRecorderItem {
+    type: 'screenshot';
+    ts: number;
+    screenshot?: string;
+    timing?: string;
+}
+
+declare interface ExecutionRecorderItem_2 {
+    type: 'screenshot';
+    ts: number;
+    screenshot?: string;
+    timing?: string;
+}
+
+export declare type ExecutionTask<E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<any, any, any>> = E & ExecutionTaskReturn<E extends ExecutionTaskApply<any, any, infer TaskOutput, any> ? TaskOutput : unknown, E extends ExecutionTaskApply<any, any, any, infer TaskLog> ? TaskLog : unknown> & {
+    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';
+    error?: Error;
+    errorMessage?: string;
+    errorStack?: string;
+    timing?: {
+        start: number;
+        end?: number;
+        cost?: number;
+    };
+    usage?: AIUsageInfo;
+};
+
+declare type ExecutionTask_2<
+E extends ExecutionTaskApply_2<any, any, any> = ExecutionTaskApply_2<
+any,
+any,
+any
+>,
+> = E &
+ExecutionTaskReturn_2<
+E extends ExecutionTaskApply_2<any, any, infer TaskOutput, any>
+? TaskOutput
+: unknown,
+E extends ExecutionTaskApply_2<any, any, any, infer TaskLog>
+? TaskLog
+: unknown
+> & {
+    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';
+    error?: Error;
+    errorMessage?: string;
+    errorStack?: string;
+    timing?: {
+        start: number;
+        end?: number;
+        cost?: number;
+    };
+    usage?: AIUsageInfo_2;
+};
+
+export declare type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;
+
+export declare type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<'Action', ActionParam, void, void>;
+
+export declare interface ExecutionTaskApply<Type extends ExecutionTaskType = any, TaskParam = any, TaskOutput = any, TaskLog = any> {
+    type: Type;
+    subType?: string;
+    param?: TaskParam;
+    thought?: string;
+    locate?: PlanningLocateParam | null;
+    pageContext?: UIContext;
+    executor: (param: TaskParam, context: ExecutorContext) => Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void> | undefined | void;
+}
+
+declare interface ExecutionTaskApply_2<
+Type extends ExecutionTaskType_2 = any,
+TaskParam = any,
+TaskOutput = any,
+TaskLog = any,
+> {
+    type: Type;
+    subType?: string;
+    param?: TaskParam;
+    thought?: string;
+    locate?: PlanningLocateParam_2 | null;
+    pageContext?: UIContext_2;
+    executor: (
+    param: TaskParam,
+    context: ExecutorContext_2,
+    ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>
+    | Promise<ExecutionTaskReturn_2<TaskOutput, TaskLog> | undefined | void>
+    | undefined
+    | void;
+}
+
+export declare interface ExecutionTaskHitBy {
+    from: string;
+    context: Record<string, any>;
+}
+
+declare interface ExecutionTaskHitBy_2 {
+    from: string;
+    context: Record<string, any>;
+}
+
+export declare type ExecutionTaskInsightAssertion = ExecutionTask<ExecutionTaskInsightAssertionApply>;
+
+export declare type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightAssertionParam, InsightAssertionResponse, ExecutionTaskInsightDumpLog>;
+
+export declare interface ExecutionTaskInsightAssertionParam {
+    assertion: string;
+}
+
+export declare interface ExecutionTaskInsightDumpLog {
+    dump?: InsightDump;
+}
+
+export declare type ExecutionTaskInsightLocate = ExecutionTask<ExecutionTaskInsightLocateApply>;
+
+export declare type ExecutionTaskInsightLocateApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightLocateParam, ExecutionTaskInsightLocateOutput, ExecutionTaskInsightDumpLog>;
+
+export declare interface ExecutionTaskInsightLocateOutput {
+    element: LocateResultElement | null;
+}
+
+export declare type ExecutionTaskInsightLocateParam = PlanningLocateParam;
+
+export declare type ExecutionTaskInsightQuery = ExecutionTask<ExecutionTaskInsightQueryApply>;
+
+export declare type ExecutionTaskInsightQueryApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightQueryParam, any, ExecutionTaskInsightDumpLog>;
+
+export declare interface ExecutionTaskInsightQueryOutput {
+    data: any;
+}
+
+export declare interface ExecutionTaskInsightQueryParam {
+    dataDemand: InsightExtractParam;
+}
+
+export declare type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;
+
+export declare type ExecutionTaskLogApply<LogParam = {
+    content: string;
+}> = ExecutionTaskApply<'Log', LogParam, void, void>;
+
+export declare type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;
+
+export declare type ExecutionTaskPlanningApply = ExecutionTaskApply<'Planning', {
+    userInstruction: string;
+    log?: string;
+}, PlanningAIResponse>;
+
+export declare interface ExecutionTaskProgressOptions {
+    onTaskStart?: (task: ExecutionTask) => Promise<void> | void;
+}
+
+declare interface ExecutionTaskProgressOptions_2 {
+    onTaskStart?: (task: ExecutionTask_2) => Promise<void> | void;
+}
+
+export declare interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {
+    output?: TaskOutput;
+    log?: TaskLog;
+    recorder?: ExecutionRecorderItem[];
+    hitBy?: ExecutionTaskHitBy;
+}
+
+declare interface ExecutionTaskReturn_2<TaskOutput = unknown, TaskLog = unknown> {
+    output?: TaskOutput;
+    log?: TaskLog;
+    recorder?: ExecutionRecorderItem_2[];
+    hitBy?: ExecutionTaskHitBy_2;
+}
+
+export declare type ExecutionTaskType = 'Planning' | 'Insight' | 'Action' | 'Assertion' | 'Log';
+
+declare type ExecutionTaskType_2 =
+| 'Planning'
+| 'Insight'
+| 'Action'
+| 'Assertion'
+| 'Log';
+
+export declare class Executor {
+    name: string;
+    tasks: ExecutionTask_2[];
+    status: 'init' | 'pending' | 'running' | 'completed' | 'error';
+    onTaskStart?: ExecutionTaskProgressOptions_2['onTaskStart'];
+    constructor(name: string, options?: ExecutionTaskProgressOptions_2 & {
+        tasks?: ExecutionTaskApply_2[];
+    });
+    private markTaskAsPending;
+    append(task: ExecutionTaskApply_2[] | ExecutionTaskApply_2): Promise<void>;
+    flush(): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
+    isInErrorState(): boolean;
+    latestErrorTask(): ExecutionTask_2 | null;
+    dump(): ExecutionDump_2;
+}
+
+export declare interface ExecutorContext {
+    task: ExecutionTask;
+    element?: LocateResultElement | null;
+}
+
+declare interface ExecutorContext_2 {
+    task: ExecutionTask_2;
+    element?: LocateResultElement_2 | null;
+}
+
+export declare interface FreeFn {
+    name: string;
+    fn: () => void;
+}
+
+export { getAIConfig }
+
+export declare function getVersion(): string;
+
+export declare interface GroupedActionDump {
+    groupName: string;
+    groupDescription?: string;
+    executions: ExecutionDump[];
+}
+
+declare class Insight<ElementType extends BaseElement = BaseElement, ContextType extends UIContext_2<ElementType> = UIContext_2<ElementType>> {
+    contextRetrieverFn: (action: InsightAction_2) => Promise<ContextType> | ContextType;
+    aiVendorFn: (...args: Array<any>) => Promise<any>;
+    onceDumpUpdatedFn?: DumpSubscriber_2;
+    taskInfo?: Omit<InsightTaskInfo_2, 'durationMs'>;
+    constructor(context: ContextType | ((action: InsightAction_2) => Promise<ContextType> | ContextType), opt?: InsightOptions_2);
+    locate(query: DetailedLocateParam_2, opt?: LocateOpts): Promise<LocateResult_2>;
+    extract<T>(dataDemand: InsightExtractParam_2, opt?: InsightExtractOption_2, multimodalPrompt?: TMultimodalPrompt_2): Promise<{
+        data: T;
+        thought?: string;
+        usage?: AIUsageInfo_2;
+    }>;
+    assert(assertion: TUserPrompt_2): Promise<InsightAssertionResponse_2>;
+    describe(target: Rect | [number, number], opt?: {
+        deepThink?: boolean;
+    }): Promise<Pick<AIDescribeElementResponse_2, 'description'>>;
+}
+export { Insight }
+export default Insight;
+
+export declare type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';
+
+declare type InsightAction_2 = 'locate' | 'extract' | 'assert' | 'describe';
+
+export declare type InsightAssertionResponse = AIAssertionResponse & {
+    usage?: AIUsageInfo;
+};
+
+declare type InsightAssertionResponse_2 = AIAssertionResponse_2 & {
+    usage?: AIUsageInfo_2;
+};
+
+export declare interface InsightDump extends DumpMeta {
+    type: 'locate' | 'extract' | 'assert';
+    logId: string;
+    userQuery: {
+        element?: TUserPrompt;
+        dataDemand?: InsightExtractParam;
+        assertion?: TUserPrompt;
+    };
+    matchedElement: BaseElement[];
+    matchedRect?: Rect;
+    deepThink?: boolean;
+    data: any;
+    assertionPass?: boolean;
+    assertionThought?: string;
+    taskInfo: InsightTaskInfo;
+    error?: string;
+    output?: any;
+}
+
+declare interface InsightDump_2 extends DumpMeta_2 {
+    type: 'locate' | 'extract' | 'assert';
+    logId: string;
+    userQuery: {
+        element?: TUserPrompt_2;
+        dataDemand?: InsightExtractParam_2;
+        assertion?: TUserPrompt_2;
+    };
+    matchedElement: BaseElement[];
+    matchedRect?: Rect;
+    deepThink?: boolean;
+    data: any;
+    assertionPass?: boolean;
+    assertionThought?: string;
+    taskInfo: InsightTaskInfo_2;
+    error?: string;
+    output?: any;
+}
+
+export declare interface InsightExtractOption {
+    domIncluded?: boolean | 'visible-only';
+    screenshotIncluded?: boolean;
+    returnThought?: boolean;
+}
+
+declare interface InsightExtractOption_2 {
+    domIncluded?: boolean | 'visible-only';
+    screenshotIncluded?: boolean;
+    returnThought?: boolean;
+}
+
+export declare type InsightExtractParam = string | Record<string, string>;
+
+declare type InsightExtractParam_2 = string | Record<string, string>;
+
+export declare interface InsightOptions {
+    taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
+    aiVendorFn?: CallAIFn;
+}
+
+declare interface InsightOptions_2 {
+    taskInfo?: Omit<InsightTaskInfo_2, 'durationMs'>;
+    aiVendorFn?: CallAIFn_2;
+}
+
+export declare interface InsightTaskInfo {
+    durationMs: number;
+    formatResponse?: string;
+    rawResponse?: string;
+    usage?: AIUsageInfo;
+    searchArea?: Rect;
+    searchAreaRawResponse?: string;
+    searchAreaUsage?: AIUsageInfo;
+}
+
+declare interface InsightTaskInfo_2 {
+    durationMs: number;
+    formatResponse?: string;
+    rawResponse?: string;
+    usage?: AIUsageInfo_2;
+    searchArea?: Rect;
+    searchAreaRawResponse?: string;
+    searchAreaUsage?: AIUsageInfo_2;
+}
+
+export declare interface LiteUISection {
+    name: string;
+    description: string;
+    sectionCharacteristics: string;
+    textIds: string[];
+}
+
+export declare interface LocateOption {
+    deepThink?: boolean;
+    cacheable?: boolean;
+    xpath?: string;
+    pageContext?: UIContext<BaseElement>;
+}
+
+declare interface LocateOption_2 {
+    deepThink?: boolean; // only available in vl model
+    cacheable?: boolean; // user can set this param to false to disable the cache for a single agent api
+    xpath?: string; // only available in web
+    pageContext?: UIContext_2<BaseElement>;
+}
+
+declare interface LocateOpts {
+    context?: UIContext_2<BaseElement>;
+    callAI?: typeof callAiFn_2<AIElementResponse_2>;
+}
+
+export declare interface LocateResult {
+    element: LocateResultElement | null;
+    rect?: Rect;
+}
+
+declare interface LocateResult_2 {
+    element: LocateResultElement_2 | null;
+    rect?: Rect;
+}
+
+export declare type LocateResultElement = {
+    id: string;
+    indexId?: number;
+    center: [number, number];
+    rect: Rect;
+    xpaths: string[];
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    isOrderSensitive?: boolean;
+};
+
+declare type LocateResultElement_2 = {
+    id: string;
+    indexId?: number;
+    center: [number, number];
+    rect: Rect;
+    xpaths: string[];
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    isOrderSensitive?: boolean;
+};
+
+export declare interface LocateValidatorResult {
+    pass: boolean;
+    rect: Rect;
+    center: [number, number];
+    centerDistance?: number;
+}
+
+export declare interface LocatorValidatorOption {
+    centerDistanceThreshold?: number;
+}
+
+export { MIDSCENE_MODEL_NAME }
+
+export declare interface MidsceneYamlConfig {
+    concurrent?: number;
+    continueOnError?: boolean;
+    summary?: string;
+    shareBrowserContext?: boolean;
+    web?: MidsceneYamlScriptWebEnv;
+    android?: MidsceneYamlScriptAndroidEnv;
+    files: string[];
+    headed?: boolean;
+    keepWindow?: boolean;
+    dotenvOverride?: boolean;
+    dotenvDebug?: boolean;
+}
+
+export declare interface MidsceneYamlConfigOutput {
+    format?: 'json';
+    path?: string;
+}
+
+export declare interface MidsceneYamlConfigResult {
+    file: string;
+    success: boolean;
+    executed: boolean;
+    output?: string | null;
+    report?: string | null;
+    error?: string;
+    duration?: number;
+}
+
+export declare type MidsceneYamlFlowItem = MidsceneYamlFlowItemAIAction | MidsceneYamlFlowItemAIAssert | MidsceneYamlFlowItemAIQuery | MidsceneYamlFlowItemAIWaitFor | MidsceneYamlFlowItemAITap | MidsceneYamlFlowItemAIRightClick | MidsceneYamlFlowItemAIHover | MidsceneYamlFlowItemAIInput | MidsceneYamlFlowItemAIKeyboardPress | MidsceneYamlFlowItemAIScroll | MidsceneYamlFlowItemSleep | MidsceneYamlFlowItemLogScreenshot;
+
+declare type MidsceneYamlFlowItem_2 =
+| MidsceneYamlFlowItemAIAction_2
+| MidsceneYamlFlowItemAIAssert_2
+| MidsceneYamlFlowItemAIQuery_2
+| MidsceneYamlFlowItemAIWaitFor_2
+| MidsceneYamlFlowItemAITap_2
+| MidsceneYamlFlowItemAIRightClick_2
+| MidsceneYamlFlowItemAIHover_2
+| MidsceneYamlFlowItemAIInput_2
+| MidsceneYamlFlowItemAIKeyboardPress_2
+| MidsceneYamlFlowItemAIScroll_2
+| MidsceneYamlFlowItemSleep_2
+| MidsceneYamlFlowItemLogScreenshot_2;
+
+export declare interface MidsceneYamlFlowItemAIAction {
+    ai?: string;
+    aiAction?: string;
+    aiActionProgressTips?: string[];
+    cacheable?: boolean;
+}
+
+declare interface MidsceneYamlFlowItemAIAction_2 {
+    ai?: string; // this is the shortcut for aiAction
+    aiAction?: string;
+    aiActionProgressTips?: string[];
+    cacheable?: boolean;
+}
+
+export declare interface MidsceneYamlFlowItemAIAsk extends InsightExtractOption {
+    aiAsk: string;
+    name?: string;
+}
+
+export declare interface MidsceneYamlFlowItemAIAssert {
+    aiAssert: string;
+    errorMessage?: string;
+}
+
+declare interface MidsceneYamlFlowItemAIAssert_2 {
+    aiAssert: string;
+    errorMessage?: string;
+}
+
+export declare interface MidsceneYamlFlowItemAIBoolean extends InsightExtractOption {
+    aiBoolean: string;
+    name?: string;
+}
+
+export declare interface MidsceneYamlFlowItemAIHover extends LocateOption {
+    aiHover: TUserPrompt;
+}
+
+declare interface MidsceneYamlFlowItemAIHover_2 extends LocateOption_2 {
+    aiHover: TUserPrompt_2;
+}
+
+export declare interface MidsceneYamlFlowItemAIInput extends LocateOption {
+    aiInput: string;
+    locate: TUserPrompt;
+}
+
+declare interface MidsceneYamlFlowItemAIInput_2 extends LocateOption_2 {
+    aiInput: string; // value to input
+    locate: TUserPrompt_2; // where to input
+}
+
+export declare interface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {
+    aiKeyboardPress: string;
+    locate?: TUserPrompt;
+}
+
+declare interface MidsceneYamlFlowItemAIKeyboardPress_2 extends LocateOption_2 {
+    aiKeyboardPress: string;
+    locate?: TUserPrompt_2; // where to press, optional
+}
+
+export declare interface MidsceneYamlFlowItemAILocate extends LocateOption {
+    aiLocate: string;
+    name?: string;
+}
+
+export declare interface MidsceneYamlFlowItemAINumber extends InsightExtractOption {
+    aiNumber: string;
+    name?: string;
+}
+
+export declare interface MidsceneYamlFlowItemAIQuery extends InsightExtractOption {
+    aiQuery: string;
+    name?: string;
+}
+
+declare interface MidsceneYamlFlowItemAIQuery_2 extends InsightExtractOption_2 {
+    aiQuery: string;
+    name?: string;
+}
+
+export declare interface MidsceneYamlFlowItemAIRightClick extends LocateOption {
+    aiRightClick: TUserPrompt;
+}
+
+declare interface MidsceneYamlFlowItemAIRightClick_2 extends LocateOption_2 {
+    aiRightClick: TUserPrompt_2;
+}
+
+export declare interface MidsceneYamlFlowItemAIScroll extends LocateOption, PlanningActionParamScroll {
+    aiScroll: null;
+    locate?: TUserPrompt;
+}
+
+declare interface MidsceneYamlFlowItemAIScroll_2
+extends LocateOption_2,
+PlanningActionParamScroll_2 {
+    aiScroll: null;
+    locate?: TUserPrompt_2; // which area to scroll, optional
+}
+
+export declare interface MidsceneYamlFlowItemAIString extends InsightExtractOption {
+    aiString: string;
+    name?: string;
+}
+
+export declare interface MidsceneYamlFlowItemAITap extends LocateOption {
+    aiTap: TUserPrompt;
+}
+
+declare interface MidsceneYamlFlowItemAITap_2 extends LocateOption_2 {
+    aiTap: TUserPrompt_2;
+}
+
+export declare interface MidsceneYamlFlowItemAIWaitFor {
+    aiWaitFor: string;
+    timeout?: number;
+}
+
+declare interface MidsceneYamlFlowItemAIWaitFor_2 {
+    aiWaitFor: string;
+    timeout?: number;
+}
+
+export declare interface MidsceneYamlFlowItemEvaluateJavaScript {
+    javascript: string;
+    name?: string;
+}
+
+export declare interface MidsceneYamlFlowItemLogScreenshot {
+    logScreenshot?: string;
+    content?: string;
+}
+
+declare interface MidsceneYamlFlowItemLogScreenshot_2 {
+    logScreenshot?: string; // optional, the title of the screenshot
+    content?: string;
+}
+
+export declare interface MidsceneYamlFlowItemSleep {
+    sleep: number;
+}
+
+declare interface MidsceneYamlFlowItemSleep_2 {
+    sleep: number;
+}
+
+export declare interface MidsceneYamlScript {
+    target?: MidsceneYamlScriptWebEnv;
+    web?: MidsceneYamlScriptWebEnv;
+    android?: MidsceneYamlScriptAndroidEnv;
+    tasks: MidsceneYamlTask[];
+}
+
+export declare interface MidsceneYamlScriptAndroidEnv extends MidsceneYamlScriptEnvBase {
+    deviceId?: string;
+    launch?: string;
+}
+
+export declare type MidsceneYamlScriptEnv = MidsceneYamlScriptWebEnv | MidsceneYamlScriptAndroidEnv;
+
+export declare interface MidsceneYamlScriptEnvBase {
+    output?: string;
+    unstableLogContent?: boolean | string;
+    aiActionContext?: string;
+}
+
+export declare interface MidsceneYamlScriptWebEnv extends MidsceneYamlScriptEnvBase {
+    serve?: string;
+    url: string;
+    userAgent?: string;
+    acceptInsecureCerts?: boolean;
+    viewportWidth?: number;
+    viewportHeight?: number;
+    viewportScale?: number;
+    waitForNetworkIdle?: {
+        timeout?: number;
+        continueOnNetworkIdleError?: boolean;
+    };
+    cookie?: string;
+    forceSameTabNavigation?: boolean;
+    bridgeMode?: false | 'newTabWithUrl' | 'currentTab';
+    closeNewTabsAfterDisconnect?: boolean;
+}
+
+export declare interface MidsceneYamlTask {
+    name: string;
+    flow: MidsceneYamlFlowItem[];
+    continueOnError?: boolean;
+}
+
+/**
+ * agent
+ */
+export declare type OnTaskStartTip = (tip: string) => Promise<void> | void;
+
+export declare type PageType = 'puppeteer' | 'playwright' | 'static' | 'chrome-extension-proxy' | 'android';
+
+declare type PageType_2 =
+| 'puppeteer'
+| 'playwright'
+| 'static'
+| 'chrome-extension-proxy'
+| 'android';
+
+export declare type PartialInsightDumpFromSDK = Omit<InsightDump, 'sdkVersion' | 'logTime' | 'logId' | 'model_name'>;
+
+export declare function plan(userInstruction: string, opts: {
+    context: UIContext_2;
+    pageType: PageType_2;
+    actionSpace: DeviceAction_2[];
+    callAI?: typeof callAiFn<PlanningAIResponse_2>;
+    log?: string;
+    actionContext?: string;
+}): Promise<PlanningAIResponse_2>;
+
+export declare interface PlanningAction<ParamType = any> {
+    thought?: string;
+    type: 'Locate' | 'Tap' | 'RightClick' | 'Hover' | 'Drag' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Error' | 'Assert' | 'AssertWithoutThrow' | 'Sleep' | 'Finished' | 'AndroidBackButton' | 'AndroidHomeButton' | 'AndroidRecentAppsButton' | 'AndroidLongPress' | 'AndroidPull';
+    param: ParamType;
+    locate?: PlanningLocateParam | null;
+}
+
+declare interface PlanningAction_2<ParamType = any> {
+    thought?: string;
+    type:
+    | 'Locate'
+    | 'Tap'
+    | 'RightClick'
+    | 'Hover'
+    | 'Drag'
+    | 'Input'
+    | 'KeyboardPress'
+    | 'Scroll'
+    | 'Error'
+    | 'Assert'
+    | 'AssertWithoutThrow'
+    | 'Sleep'
+    | 'Finished'
+    | 'AndroidBackButton'
+    | 'AndroidHomeButton'
+    | 'AndroidRecentAppsButton'
+    | 'AndroidLongPress'
+    | 'AndroidPull';
+    param: ParamType;
+    locate?: PlanningLocateParam_2 | null;
+}
+
+export declare interface PlanningActionParamAndroidLongPress {
+    x: number;
+    y: number;
+    duration?: number;
+}
+
+export declare interface PlanningActionParamAndroidPull {
+    direction: 'up' | 'down';
+    startPoint?: {
+        x: number;
+        y: number;
+    };
+    distance?: number;
+    duration?: number;
+}
+
+export declare interface PlanningActionParamAssert {
+    assertion: TUserPrompt;
+}
+
+export declare interface PlanningActionParamError {
+    thought: string;
+}
+
+export declare type PlanningActionParamHover = null;
+
+export declare interface PlanningActionParamInputOrKeyPress {
+    value: string;
+    autoDismissKeyboard?: boolean;
+}
+
+export declare type PlanningActionParamRightClick = null;
+
+export declare type PlanningActionParamScroll = scrollParam;
+
+declare type PlanningActionParamScroll_2 = scrollParam_2;
+
+export declare interface PlanningActionParamSleep {
+    timeMs: number;
+}
+
+export declare type PlanningActionParamTap = null;
+
+export declare type PlanningActionParamWaitFor = AgentWaitForOpt & {
+    assertion: string;
+};
+
+export declare interface PlanningAIResponse {
+    action?: PlanningAction;
+    actions?: PlanningAction[];
+    more_actions_needed_by_instruction: boolean;
+    log: string;
+    sleep?: number;
+    error?: string;
+    usage?: AIUsageInfo;
+    rawResponse?: string;
+    yamlFlow?: MidsceneYamlFlowItem[];
+    yamlString?: string;
+}
+
+declare interface PlanningAIResponse_2 {
+    action?: PlanningAction_2; // this is the qwen mode
+    actions?: PlanningAction_2[];
+    more_actions_needed_by_instruction: boolean;
+    log: string;
+    sleep?: number;
+    error?: string;
+    usage?: AIUsageInfo_2;
+    rawResponse?: string;
+    yamlFlow?: MidsceneYamlFlowItem_2[];
+    yamlString?: string;
+}
+
+/**
+ * planning
+ *
+ */
+export declare interface PlanningLocateParam extends DetailedLocateParam {
+    id?: string;
+    bbox?: [number, number, number, number];
+}
+
+/**
+ * planning
+ *
+ */
+
+declare interface PlanningLocateParam_2 extends DetailedLocateParam_2 {
+    id?: string;
+    bbox?: [number, number, number, number];
+}
+
+export declare interface PlaywrightParserOpt extends BaseAgentParserOpt {
+}
+
+export { Point }
+
+export declare interface PuppeteerParserOpt extends BaseAgentParserOpt {
+}
+
+export { Rect }
+
+export declare interface ReferenceImage {
+    base64: string;
+    rect?: Rect;
+}
+
+declare interface ReferenceImage_2 {
+    base64: string;
+    rect?: Rect;
+}
+
+export declare interface ReportDumpWithAttributes {
+    dumpString: string;
+    attributes?: Record<string, any>;
+}
+
+export declare type ScriptPlayerStatusValue = 'init' | 'running' | 'done' | 'error';
+
+export declare interface ScriptPlayerTaskStatus extends MidsceneYamlTask {
+    status: ScriptPlayerStatusValue;
+    currentStep?: number;
+    totalSteps: number;
+    error?: Error;
+}
+
+export declare interface scrollParam {
+    direction: 'down' | 'up' | 'right' | 'left';
+    scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
+    distance?: null | number;
+}
+
+declare interface scrollParam_2 {
+    direction: 'down' | 'up' | 'right' | 'left';
+    scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
+    distance?: null | number; // distance in px
+}
+
+export { Size }
+
+export declare interface StreamingAIResponse {
+    /** The final accumulated content */
+    content: string;
+    /** Token usage information */
+    usage?: AIUsageInfo;
+    /** Whether the response was streamed */
+    isStreamed: boolean;
+}
+
+export declare type StreamingCallback = (chunk: CodeGenerationChunk) => void;
+
+export declare interface StreamingCodeGenerationOptions {
+    /** Whether to enable streaming output */
+    stream?: boolean;
+    /** Callback function to handle streaming chunks */
+    onChunk?: StreamingCallback;
+    /** Callback function to handle streaming completion */
+    onComplete?: (finalCode: string) => void;
+    /** Callback function to handle streaming errors */
+    onError?: (error: Error) => void;
+}
+
+export declare type TMultimodalPrompt = {
+    /**
+     * Support use image to inspect elements.
+     * The "images" field is an object that uses image name as key and image url as value.
+     * The image url can be a local path, a http link , or a base64 string.
+     */
+    images?: {
+        name: string;
+        url: string;
+    }[];
+    /**
+     * By default, the image url in the "images" filed starts with `https://` or `http://` will be directly sent to the LLM.
+     * In case the images are not accessible to the LLM (One common case is that image url is internal network only.), you can enable this option.
+     * Then image will be download and convert to base64 format.
+     */
+    convertHttpImage2Base64?: boolean;
+};
+
+declare type TMultimodalPrompt_2 = {
+    /**
+     * Support use image to inspect elements.
+     * The "images" field is an object that uses image name as key and image url as value.
+     * The image url can be a local path, a http link , or a base64 string.
+     */
+    images?: {
+        name: string;
+        url: string;
+    }[];
+    /**
+     * By default, the image url in the "images" filed starts with `https://` or `http://` will be directly sent to the LLM.
+     * In case the images are not accessible to the LLM (One common case is that image url is internal network only.), you can enable this option.
+     * Then image will be download and convert to base64 format.
+     */
+    convertHttpImage2Base64?: boolean;
+};
+
+export declare type TUserPrompt = string | ({
+    prompt: string;
+} & Partial<TMultimodalPrompt>);
+
+declare type TUserPrompt_2 =
+| string
+| ({
+    prompt: string;
+} & Partial<TMultimodalPrompt_2>);
+
+/**
+ * context
+ */
+export declare abstract class UIContext<ElementType extends BaseElement = BaseElement> {
+    abstract screenshotBase64: string;
+    abstract tree: ElementTreeNode<ElementType>;
+    abstract size: Size;
+}
+
+/**
+ * context
+ */
+
+declare abstract class UIContext_2<ElementType extends BaseElement = BaseElement> {
+    abstract screenshotBase64: string;
+
+    abstract tree: ElementTreeNode<ElementType>;
+
+    abstract size: Size;
+}
+
+export { }