@aiscene/core 1.1.1

package/dist/types/types.d.ts

@@ -0,0 +1,681 @@
+import type { NodeType } from '@midscene/shared/constants';
+import type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';
+import type { BaseElement, LocateResultElement, Rect, Size } from '@midscene/shared/types';
+import type { z } from 'zod';
+import type { TUserPrompt } from './common';
+import { ScreenshotItem } from './screenshot-item';
+import type { DetailedLocateParam, MidsceneYamlFlowItem, ServiceExtractOption } from './yaml';
+export type { ElementTreeNode, BaseElement, Rect, Size, Point, } from '@midscene/shared/types';
+export * from './yaml';
+export type AIUsageInfo = Record<string, any> & {
+    prompt_tokens: number | undefined;
+    completion_tokens: number | undefined;
+    total_tokens: number | undefined;
+    cached_input: number | undefined;
+    time_cost: number | undefined;
+    model_name: string | undefined;
+    model_description: string | undefined;
+    intent: string | undefined;
+    request_id: string | undefined;
+};
+export type { LocateResultElement };
+export type AISingleElementResponseByPosition = {
+    position?: {
+        x: number;
+        y: number;
+    };
+    bbox?: [number, number, number, number];
+    reason: string;
+    text: string;
+};
+export interface AIElementCoordinatesResponse {
+    bbox: [number, number, number, number];
+    errors?: string[];
+}
+export type AIElementResponse = AIElementCoordinatesResponse;
+export interface AIDataExtractionResponse<DataDemand> {
+    data: DataDemand;
+    errors?: string[];
+    thought?: string;
+}
+export interface AISectionLocatorResponse {
+    bbox: [number, number, number, number];
+    references_bbox?: [number, number, number, number][];
+    error?: string;
+}
+export interface AIAssertionResponse {
+    pass: boolean;
+    thought: string;
+}
+export interface AIDescribeElementResponse {
+    description: string;
+    error?: string;
+}
+export interface LocatorValidatorOption {
+    centerDistanceThreshold?: number;
+}
+export interface LocateValidatorResult {
+    pass: boolean;
+    rect: Rect;
+    center: [number, number];
+    centerDistance?: number;
+}
+export interface AgentDescribeElementAtPointResult {
+    prompt: string;
+    deepLocate: boolean;
+    verifyResult?: LocateValidatorResult;
+}
+/**
+ * context
+ */
+export declare abstract class UIContext {
+    /**
+     * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),
+     */
+    abstract screenshot: ScreenshotItem;
+    /**
+     * screenshot size after shrinking
+     */
+    abstract shotSize: Size;
+    /**
+     * The ratio for converting shrunk screenshot coordinates to logical coordinates.
+     *
+     * Example:
+     * - Physical screen width: 3000px, dpr=6
+     * - Logical width: 500px
+     * - User-defined screenshotShrinkFactor: 2
+     * - Actual shrunk screenshot width: 3000 / 2 = 1500px
+     * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3
+     * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px
+     */
+    abstract shrunkShotToLogicalRatio: number;
+    abstract _isFrozen?: boolean;
+    abstract deprecatedDpr?: number;
+}
+export type EnsureObject<T> = {
+    [K in keyof T]: any;
+};
+export type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';
+export type ServiceExtractParam = string | Record<string, string>;
+export type ElementCacheFeature = Record<string, unknown>;
+export interface LocateResult {
+    element: LocateResultElement | null;
+    rect?: Rect;
+}
+export type ThinkingLevel = 'off' | 'medium' | 'high';
+export type DeepThinkOption = 'unset' | true | false;
+export interface ServiceTaskInfo {
+    durationMs: number;
+    formatResponse?: string;
+    rawResponse?: string;
+    usage?: AIUsageInfo;
+    searchArea?: Rect;
+    searchAreaRawResponse?: string;
+    searchAreaUsage?: AIUsageInfo;
+    reasoning_content?: string;
+}
+export interface DumpMeta {
+    logTime: number;
+}
+export type ReportAttributes = Record<string, string | number | boolean | null | undefined>;
+export interface ReportDumpWithAttributes {
+    dumpString: string;
+    attributes?: ReportAttributes;
+}
+export interface ServiceDump extends DumpMeta {
+    type: 'locate' | 'extract' | 'assert';
+    logId: string;
+    userQuery: {
+        element?: TUserPrompt;
+        dataDemand?: ServiceExtractParam;
+        assertion?: TUserPrompt;
+    };
+    matchedElement: LocateResultElement[];
+    matchedRect?: Rect;
+    deepLocate?: boolean;
+    data: any;
+    assertionPass?: boolean;
+    assertionThought?: string;
+    taskInfo: ServiceTaskInfo;
+    error?: string;
+    output?: any;
+}
+export type PartialServiceDumpFromSDK = Omit<ServiceDump, 'logTime' | 'logId' | 'model_name'>;
+export interface ServiceResultBase {
+    dump: ServiceDump;
+}
+export type LocateResultWithDump = LocateResult & ServiceResultBase;
+export interface ServiceExtractResult<T> extends ServiceResultBase {
+    data: T;
+    thought?: string;
+    usage?: AIUsageInfo;
+    reasoning_content?: string;
+}
+export declare class ServiceError extends Error {
+    dump: ServiceDump;
+    constructor(message: string, dump: ServiceDump);
+}
+export interface LiteUISection {
+    name: string;
+    description: string;
+    sectionCharacteristics: string;
+    textIds: string[];
+}
+export type ElementById = (id: string) => BaseElement | null;
+export type ServiceAssertionResponse = AIAssertionResponse & {
+    usage?: AIUsageInfo;
+};
+/**
+ * agent
+ */
+export type OnTaskStartTip = (tip: string) => Promise<void> | void;
+export interface AgentWaitForOpt extends ServiceExtractOption {
+    checkIntervalMs?: number;
+    timeoutMs?: number;
+}
+export interface AgentAssertOpt {
+    keepRawResponse?: boolean;
+}
+/**
+ * planning
+ *
+ */
+export interface PlanningLocateParam extends DetailedLocateParam {
+    bbox?: [number, number, number, number];
+}
+export interface PlanningAction<ParamType = any> {
+    thought?: string;
+    log?: string;
+    type: string;
+    param: ParamType;
+}
+export type SubGoalStatus = 'pending' | 'running' | 'finished';
+export interface SubGoal {
+    index: number;
+    status: SubGoalStatus;
+    description: string;
+    logs?: string[];
+}
+export interface RawResponsePlanningAIResponse {
+    action: PlanningAction;
+    thought?: string;
+    log: string;
+    memory?: string;
+    error?: string;
+    finalizeMessage?: string;
+    finalizeSuccess?: boolean;
+    updateSubGoals?: SubGoal[];
+    markFinishedIndexes?: number[];
+}
+export interface PlanningAIResponse extends Omit<RawResponsePlanningAIResponse, 'action'> {
+    actions?: PlanningAction[];
+    usage?: AIUsageInfo;
+    rawResponse?: string;
+    yamlFlow?: MidsceneYamlFlowItem[];
+    yamlString?: string;
+    error?: string;
+    reasoning_content?: string;
+    shouldContinuePlanning: boolean;
+    output?: string;
+}
+export interface PlanningActionParamSleep {
+    timeMs: number;
+}
+export interface PlanningActionParamError {
+    thought: string;
+}
+export type PlanningActionParamWaitFor = AgentWaitForOpt & {};
+export interface LongPressParam {
+    duration?: number;
+}
+export interface PullParam {
+    direction: 'up' | 'down';
+    distance?: number;
+    duration?: number;
+}
+/**
+ * misc
+ */
+export interface Color {
+    name: string;
+    hex: string;
+}
+export interface BaseAgentParserOpt {
+    selector?: string;
+}
+export interface PuppeteerParserOpt extends BaseAgentParserOpt {
+}
+export interface PlaywrightParserOpt extends BaseAgentParserOpt {
+}
+export interface ExecutionTaskProgressOptions {
+    onTaskStart?: (task: ExecutionTask) => Promise<void> | void;
+}
+export interface ExecutionRecorderItem {
+    type: 'screenshot';
+    ts: number;
+    screenshot?: ScreenshotItem;
+    timing?: string;
+}
+export type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';
+export interface ExecutorContext {
+    task: ExecutionTask;
+    element?: LocateResultElement | null;
+    uiContext?: UIContext;
+}
+export interface ExecutionTaskApply<Type extends ExecutionTaskType = any, TaskParam = any, TaskOutput = any, TaskLog = any> {
+    type: Type;
+    subType?: string;
+    param?: TaskParam;
+    thought?: string;
+    uiContext?: UIContext;
+    executor: (param: TaskParam, context: ExecutorContext) => Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void> | undefined | void;
+}
+export interface ExecutionTaskHitBy {
+    from: string;
+    context: Record<string, any>;
+}
+export interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {
+    output?: TaskOutput;
+    log?: TaskLog;
+    recorder?: ExecutionRecorderItem[];
+    hitBy?: ExecutionTaskHitBy;
+}
+export type ExecutionTask<E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<any, any, any>> = E & ExecutionTaskReturn<E extends ExecutionTaskApply<any, any, infer TaskOutput, any> ? TaskOutput : unknown, E extends ExecutionTaskApply<any, any, any, infer TaskLog> ? TaskLog : unknown> & {
+    taskId: string;
+    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';
+    error?: Error;
+    errorMessage?: string;
+    errorStack?: string;
+    timing?: {
+        start: number;
+        getUiContextStart?: number;
+        getUiContextEnd?: number;
+        callAiStart?: number;
+        callAiEnd?: number;
+        beforeInvokeActionHookStart?: number;
+        beforeInvokeActionHookEnd?: number;
+        callActionStart?: number;
+        callActionEnd?: number;
+        afterInvokeActionHookStart?: number;
+        afterInvokeActionHookEnd?: number;
+        captureAfterCallingSnapshotStart?: number;
+        captureAfterCallingSnapshotEnd?: number;
+        end?: number;
+        cost?: number;
+    };
+    usage?: AIUsageInfo;
+    searchAreaUsage?: AIUsageInfo;
+    reasoning_content?: string;
+};
+export interface IExecutionDump extends DumpMeta {
+    /** Stable unique identifier for this execution run */
+    id?: string;
+    name: string;
+    description?: string;
+    tasks: ExecutionTask[];
+    aiActContext?: string;
+}
+/**
+ * ExecutionDump class for serializing and deserializing execution dumps
+ */
+export declare class ExecutionDump implements IExecutionDump {
+    id?: string;
+    logTime: number;
+    name: string;
+    description?: string;
+    tasks: ExecutionTask[];
+    aiActContext?: string;
+    constructor(data: IExecutionDump);
+    /**
+     * Serialize the ExecutionDump to a JSON string
+     */
+    serialize(indents?: number): string;
+    /**
+     * Convert to a plain object for JSON serialization
+     */
+    toJSON(): IExecutionDump;
+    /**
+     * Create an ExecutionDump instance from a serialized JSON string
+     */
+    static fromSerializedString(serialized: string): ExecutionDump;
+    /**
+     * Create an ExecutionDump instance from a plain object
+     */
+    static fromJSON(data: IExecutionDump): ExecutionDump;
+    /**
+     * Collect all ScreenshotItem instances from tasks.
+     * Scans through uiContext and recorder items to find screenshots.
+     *
+     * @returns Array of ScreenshotItem instances
+     */
+    collectScreenshots(): ScreenshotItem[];
+}
+export type ExecutionTaskInsightLocateParam = PlanningLocateParam;
+export interface ExecutionTaskInsightLocateOutput {
+    element: LocateResultElement | null;
+}
+export type ExecutionTaskInsightDump = ServiceDump;
+export type ExecutionTaskInsightLocateApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightLocateParam, ExecutionTaskInsightLocateOutput, ExecutionTaskInsightDump>;
+export type ExecutionTaskInsightLocate = ExecutionTask<ExecutionTaskInsightLocateApply>;
+export interface ExecutionTaskInsightQueryParam {
+    dataDemand: ServiceExtractParam;
+    domIncluded?: boolean | 'visible-only';
+}
+export interface ExecutionTaskInsightQueryOutput {
+    data: any;
+}
+export type ExecutionTaskInsightQueryApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightQueryParam, any, ExecutionTaskInsightDump>;
+export type ExecutionTaskInsightQuery = ExecutionTask<ExecutionTaskInsightQueryApply>;
+export interface ExecutionTaskInsightAssertionParam {
+    assertion: string;
+}
+export type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightAssertionParam, ServiceAssertionResponse, ExecutionTaskInsightDump>;
+export type ExecutionTaskInsightAssertion = ExecutionTask<ExecutionTaskInsightAssertionApply>;
+export type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<'Action Space', ActionParam, void, void>;
+export type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;
+export type ExecutionTaskLogApply<LogParam = {
+    content: string;
+}> = ExecutionTaskApply<'Log', LogParam, void, void>;
+export type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;
+export type ExecutionTaskPlanningApply = ExecutionTaskApply<'Planning', {
+    userInstruction: string;
+    aiActContext?: string;
+}, PlanningAIResponse>;
+export type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;
+export type ExecutionTaskPlanningLocateParam = PlanningLocateParam;
+export interface ExecutionTaskPlanningLocateOutput {
+    element: LocateResultElement | null;
+}
+export type ExecutionTaskPlanningDump = ServiceDump;
+export type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<'Planning', ExecutionTaskPlanningLocateParam, ExecutionTaskPlanningLocateOutput, ExecutionTaskPlanningDump>;
+export type ExecutionTaskPlanningLocate = ExecutionTask<ExecutionTaskPlanningLocateApply>;
+export interface ReportMeta {
+    groupName: string;
+    groupDescription?: string;
+    sdkVersion: string;
+    modelBriefs: ModelBrief[];
+    deviceType?: string;
+}
+export type GroupMeta = ReportMeta;
+export interface IReportActionDump {
+    sdkVersion: string;
+    groupName: string;
+    groupDescription?: string;
+    modelBriefs: ModelBrief[];
+    executions: IExecutionDump[];
+    deviceType?: string;
+}
+export type IGroupedActionDump = IReportActionDump;
+export interface ModelBrief {
+    /**
+     * The intent/category of the model call, for example "planning" or "insight".
+     */
+    intent?: string;
+    /**
+     * The model name returned by usage metadata, for example "gpt-4o".
+     */
+    name?: string;
+    /**
+     * Optional human-readable model description, for example "qwen2.5-vl mode".
+     */
+    modelDescription?: string;
+}
+/**
+ * ReportActionDump class for serializing and deserializing report action dumps
+ */
+export declare class ReportActionDump implements IReportActionDump {
+    sdkVersion: string;
+    groupName: string;
+    groupDescription?: string;
+    modelBriefs: ModelBrief[];
+    executions: ExecutionDump[];
+    deviceType?: string;
+    constructor(data: IReportActionDump);
+    /**
+     * Serialize the ReportActionDump to a JSON string
+     * Uses compact { $screenshot: id } format
+     */
+    serialize(indents?: number): string;
+    /**
+     * Serialize the ReportActionDump with inline screenshots to a JSON string.
+     * Each ScreenshotItem is replaced with { base64: "...", capturedAt }.
+     */
+    serializeWithInlineScreenshots(indents?: number): string;
+    /**
+     * Convert to a plain object for JSON serialization
+     */
+    toJSON(): IReportActionDump;
+    /**
+     * Create a ReportActionDump instance from a serialized JSON string
+     */
+    static fromSerializedString(serialized: string): ReportActionDump;
+    /**
+     * Create a ReportActionDump instance from a plain object
+     */
+    static fromJSON(data: IReportActionDump): ReportActionDump;
+    /**
+     * Collect all ScreenshotItem instances from all executions.
+     *
+     * @returns Array of all ScreenshotItem instances across all executions
+     */
+    collectAllScreenshots(): ScreenshotItem[];
+    /**
+     * Serialize the dump to files with screenshots as separate PNG files.
+     * Creates:
+     * - {basePath} - dump JSON with { $screenshot: id } references
+     * - {basePath}.screenshots/ - PNG files
+     *
+     * @param basePath - Base path for the dump file
+     */
+    serializeToFiles(basePath: string): void;
+    /**
+     * Read dump from files and return JSON string with inline screenshots.
+     * Reads the dump JSON and screenshot files, then inlines the base64 data.
+     *
+     * @param basePath - Base path for the dump file
+     * @returns JSON string with inline screenshots ({ base64: "..." } format)
+     */
+    static fromFilesAsInlineJson(basePath: string): string;
+    /**
+     * Clean up all files associated with a serialized dump.
+     *
+     * @param basePath - Base path for the dump file
+     */
+    static cleanupFiles(basePath: string): void;
+    /**
+     * Get all file paths associated with a serialized dump.
+     *
+     * @param basePath - Base path for the dump file
+     * @returns Array of all associated file paths
+     */
+    static getFilePaths(basePath: string): string[];
+}
+export type GroupedActionDump = ReportActionDump;
+export declare const GroupedActionDump: typeof ReportActionDump;
+export type InterfaceType = 'puppeteer' | 'playwright' | 'static' | 'chrome-extension-proxy' | 'android' | string;
+export interface StreamingCodeGenerationOptions {
+    /** Whether to enable streaming output */
+    stream?: boolean;
+    /** Callback function to handle streaming chunks */
+    onChunk?: StreamingCallback;
+    /** Callback function to handle streaming completion */
+    onComplete?: (finalCode: string) => void;
+    /** Callback function to handle streaming errors */
+    onError?: (error: Error) => void;
+}
+export type StreamingCallback = (chunk: CodeGenerationChunk) => void;
+export interface CodeGenerationChunk {
+    /** The incremental content chunk */
+    content: string;
+    /** The reasoning content */
+    reasoning_content: string;
+    /** The accumulated content so far */
+    accumulated: string;
+    /** Whether this is the final chunk */
+    isComplete: boolean;
+    /** Token usage information if available */
+    usage?: AIUsageInfo;
+}
+export interface StreamingAIResponse {
+    /** The final accumulated content */
+    content: string;
+    /** Token usage information */
+    usage?: AIUsageInfo;
+    /** Whether the response was streamed */
+    isStreamed: boolean;
+}
+export interface DeviceAction<TParam = any, TReturn = any> {
+    name: string;
+    description?: string;
+    interfaceAlias?: string;
+    paramSchema?: z.ZodType<TParam>;
+    call: (param: TParam, context: ExecutorContext) => Promise<TReturn> | TReturn;
+    delayAfterRunner?: number;
+    /**
+     * An example param object for this action.
+     * Locate fields with { prompt } will automatically get bbox injected when needed.
+     */
+    sample?: {
+        [K in keyof TParam]?: any;
+    };
+}
+/**
+ * Type utilities for extracting types from DeviceAction definitions
+ */
+/**
+ * Extract parameter type from a DeviceAction
+ */
+export type ActionParam<Action extends DeviceAction<any, any>> = Action extends DeviceAction<infer P, any> ? P : never;
+/**
+ * Extract return type from a DeviceAction
+ */
+export type ActionReturn<Action extends DeviceAction<any, any>> = Action extends DeviceAction<any, infer R> ? R : never;
+/**
+ * Web-specific types
+ */
+export interface WebElementInfo extends BaseElement {
+    id: string;
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+}
+/**
+ * Agent
+ */
+export type CacheConfig = {
+    strategy?: 'read-only' | 'read-write' | 'write-only';
+    id: string;
+};
+export type Cache = false | true | CacheConfig;
+export interface AgentOpt {
+    testId?: string;
+    cacheId?: string;
+    groupName?: string;
+    groupDescription?: string;
+    generateReport?: boolean;
+    persistExecutionDump?: boolean;
+    autoPrintReportMsg?: boolean;
+    /**
+     * Use directory-based report format with separate image files.
+     *
+     * When enabled:
+     * - Screenshots are saved as PNG files in a `screenshots/` subdirectory
+     * - Report is generated as `index.html` with relative image paths
+     * - Reduces memory usage and report file size
+     *
+     * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server
+     * (e.g., `npx serve ./report-dir`). The file:// protocol will not
+     * work due to browser CORS restrictions.
+     *
+     * @default 'single-html'
+     */
+    outputFormat?: 'single-html' | 'html-and-external-assets';
+    onTaskStartTip?: OnTaskStartTip;
+    aiActContext?: string;
+    aiActionContext?: string;
+    reportFileName?: string;
+    modelConfig?: TModelConfig;
+    cache?: Cache;
+    /**
+     * Maximum number of replanning cycles for aiAct.
+     * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.
+     * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.
+     */
+    replanningCycleLimit?: number;
+    /**
+     * Wait time in milliseconds after each action execution.
+     * This allows the UI to settle and stabilize before the next action.
+     * Defaults to 300ms when not provided.
+     */
+    waitAfterAction?: number;
+    /**
+     * When set to true, Midscene will use the target device's time (Android/iOS)
+     * instead of the system time. Useful when the device time differs from the
+     * host machine. Default: false
+     */
+    useDeviceTimestamp?: boolean;
+    /**
+     * Custom screenshot shrink factor to reduce AI token usage.
+     * When set, the screenshot will be scaled down by this factor from the physical resolution.
+     *
+     * Example:
+     * - Physical screen width: 3000px, dpr=6
+     * - Logical width: 500px
+     * - screenshotShrinkFactor: 2
+     * - Actual shrunk screenshot width: 3000 / 2 = 1500px
+     * - AI analyzes the 1500px screenshot
+     * - Coordinates are transformed back to logical (500px) before actions execute
+     *
+     * Benefits:
+     * - Reduces token usage for high-resolution screenshots
+     * - Maintains accuracy by scaling coordinates appropriately
+     *
+     * Must be >= 1 (shrinking only, enlarging is not supported).
+     *
+     * @default 1 (no shrinking, uses original physical screenshot)
+     */
+    screenshotShrinkFactor?: number;
+    /**
+     * Custom OpenAI client factory function
+     *
+     * If provided, this function will be called to create OpenAI client instances
+     * for each AI call, allowing you to:
+     * - Wrap clients with observability tools (langsmith, langfuse)
+     * - Use custom OpenAI-compatible clients
+     * - Apply different configurations based on intent
+     *
+     * @param config - Resolved model configuration
+     * @returns OpenAI client instance (original or wrapped)
+     *
+     * @example
+     * ```typescript
+     * createOpenAIClient: async (openai, opts) => {
+     *   // Wrap with langsmith for planning tasks
+     *   if (opts.baseURL?.includes('planning')) {
+     *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });
+     *   }
+     *
+     *   return openai;
+     * }
+     * ```
+     */
+    createOpenAIClient?: CreateOpenAIClientFn;
+}
+export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
+export interface ReportFileAttributes {
+    testDuration: number;
+    testStatus: TestStatus;
+    testTitle: string;
+    testId: string;
+    testDescription: string;
+}
+export type ReportFileWithAttributes = {
+    reportFilePath: string;
+    reportAttributes: ReportFileAttributes;
+} | {
+    reportFilePath?: string;
+    reportAttributes: ReportFileAttributes & {
+        testStatus: 'skipped';
+    };
+};