@godscene/core 1.7.11

package/dist/types/agent/agent.d.ts

@@ -0,0 +1,220 @@
+import type { TUserPrompt } from '../ai-model/index';
+import Service from '../service/index';
+import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type UIContext } from '../types';
+import type { AbstractInterface } from '../device';
+import type { TaskRunner } from '../task-runner';
+import { ModelConfigManager } from '@godscene/shared/env';
+import { TaskCache } from './task-cache';
+import { TaskExecutor } from './tasks';
+export type AiActOptions = {
+    cacheable?: boolean;
+    fileChooserAccept?: string | string[];
+    deepThink?: DeepThinkOption;
+    deepLocate?: boolean;
+    abortSignal?: AbortSignal;
+};
+export declare class Agent<InterfaceType extends AbstractInterface = AbstractInterface> {
+    interface: InterfaceType;
+    service: Service;
+    dump: ReportActionDump;
+    reportFile?: string | null;
+    reportFileName?: string;
+    taskExecutor: TaskExecutor;
+    opts: AgentOpt;
+    /**
+     * If true, the agent will not perform any actions
+     */
+    dryMode: boolean;
+    onTaskStartTip?: OnTaskStartTip;
+    taskCache?: TaskCache;
+    private dumpUpdateListeners;
+    get onDumpUpdate(): ((dump: string, executionDump?: ExecutionDump) => void) | undefined;
+    set onDumpUpdate(callback: ((dump: string, executionDump?: ExecutionDump) => void) | undefined);
+    destroyed: boolean;
+    modelConfigManager: ModelConfigManager;
+    /**
+     * Frozen page context for consistent AI operations
+     */
+    private frozenUIContext?;
+    private get aiActContext();
+    /**
+     * Flag to track if VL model warning has been shown
+     */
+    private hasWarnedNonVLModel;
+    private executionDumpIndexByRunner;
+    private fullActionSpace;
+    private reportGenerator;
+    get page(): InterfaceType;
+    /**
+     * Ensures VL model warning is shown once when needed
+     */
+    private ensureVLModelWarning;
+    private resolveReplanningCycleLimit;
+    constructor(interfaceInstance: InterfaceType, opts?: AgentOpt);
+    getActionSpace(): Promise<DeviceAction[]>;
+    private static readonly CONTEXT_RETRY_MAX;
+    private static readonly CONTEXT_RETRY_DELAY_MS;
+    /**
+     * Override in subclasses to indicate which errors are transient and should
+     * trigger an automatic retry when building the UI context.
+     * Returns `false` by default (no retry).
+     */
+    protected isRetryableContextError(_error: unknown): boolean;
+    getUIContext(action?: ServiceAction): Promise<UIContext>;
+    _snapshotContext(): Promise<UIContext>;
+    /**
+     * @deprecated Use {@link setAIActContext} instead.
+     */
+    setAIActionContext(prompt: string): Promise<void>;
+    setAIActContext(prompt: string): Promise<void>;
+    resetDump(): ReportActionDump;
+    appendExecutionDump(execution: ExecutionDump, runner?: TaskRunner): void;
+    dumpDataString(opt?: {
+        inlineScreenshots?: boolean;
+    }): string;
+    reportHTMLString(opt?: {
+        inlineScreenshots?: boolean;
+    }): string;
+    private lastExecutionDump?;
+    writeOutActionDumps(executionDump?: ExecutionDump): void;
+    private getReportMeta;
+    private callbackOnTaskStartTip;
+    wrapActionInActionSpace<T extends DeviceAction>(name: string): (param: ActionParam<T>) => Promise<ActionReturn<T>>;
+    callActionInActionSpace<T = any>(type: string, opt?: T): Promise<any>;
+    aiTap(locatePrompt: TUserPrompt, opt?: LocateOption & {
+        fileChooserAccept?: string | string[];
+    }): Promise<any>;
+    aiRightClick(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiDoubleClick(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiHover(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiInput(locatePrompt: TUserPrompt, opt: LocateOption & {
+        value: string | number;
+    } & {
+        autoDismissKeyboard?: boolean;
+    } & {
+        mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
+    }): Promise<any>;
+    /**
+     * @deprecated Use aiInput(locatePrompt, opt) instead where opt contains the value
+     */
+    aiInput(value: string | number, locatePrompt: TUserPrompt, opt?: LocateOption & {
+        autoDismissKeyboard?: boolean;
+    } & {
+        mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
+    }): Promise<any>;
+    aiKeyboardPress(locatePrompt: TUserPrompt, opt: LocateOption & {
+        keyName: string;
+    }): Promise<any>;
+    /**
+     * @deprecated Use aiKeyboardPress(locatePrompt, opt) instead where opt contains the keyName
+     */
+    aiKeyboardPress(keyName: string, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiScroll(locatePrompt: TUserPrompt | undefined, opt: LocateOption & ScrollParam): Promise<any>;
+    /**
+     * @deprecated Use aiScroll(locatePrompt, opt) instead where opt contains the scroll parameters
+     */
+    aiScroll(scrollParam: ScrollParam, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiPinch(locatePrompt: TUserPrompt | undefined, opt: LocateOption & {
+        direction: 'in' | 'out';
+        distance?: number;
+        duration?: number;
+    }): Promise<any>;
+    aiLongPress(locatePrompt: TUserPrompt, opt?: LocateOption & {
+        duration?: number;
+    }): Promise<any>;
+    aiClearInput(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiAct(taskPrompt: string, opt?: AiActOptions): Promise<string | undefined>;
+    /**
+     * @deprecated Use {@link Agent.aiAct} instead.
+     */
+    aiAction(taskPrompt: string, opt?: AiActOptions): Promise<string | undefined>;
+    aiQuery<ReturnType = any>(demand: ServiceExtractParam, opt?: ServiceExtractOption): Promise<ReturnType>;
+    aiBoolean(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<boolean>;
+    aiNumber(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<number>;
+    aiString(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<string>;
+    aiAsk(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<string>;
+    describeElementAtPoint(center: [number, number], opt?: {
+        verifyPrompt?: boolean;
+        retryLimit?: number;
+        deepLocate?: boolean;
+    } & LocatorValidatorOption): Promise<AgentDescribeElementAtPointResult>;
+    verifyLocator(prompt: string, locateOpt: LocateOption | undefined, expectCenter: [number, number], verifyLocateOption?: LocatorValidatorOption): Promise<LocateValidatorResult>;
+    /**
+     * Locate an element and return both its center point and an approximate rect.
+     *
+     * - In most locate flows, `rect` represents the matched element boundary.
+     * - Some models only support point grounding instead of boundary grounding.
+     *   In those cases (for example, AutoGLM), `rect` falls back to a small 8x8
+     *   box centered on the located point.
+     *
+     * Because `rect` may vary with the underlying model capability, avoid relying
+     * on it too heavily for strict boundary semantics. If you need a stable click
+     * target, prefer `center`.
+     */
+    aiLocate(prompt: TUserPrompt, opt?: LocateOption): Promise<Pick<LocateResultElement, "rect" | "center">>;
+    aiAssert(assertion: TUserPrompt, msg?: string, opt?: AgentAssertOpt & ServiceExtractOption): Promise<{
+        pass: boolean;
+        thought: string | undefined;
+        message: string | undefined;
+    } | undefined>;
+    aiWaitFor(assertion: TUserPrompt, opt?: AgentWaitForOpt): Promise<void>;
+    ai(...args: Parameters<typeof this.aiAct>): Promise<string | undefined>;
+    runYaml(yamlScriptContent: string): Promise<{
+        result: Record<string, any>;
+    }>;
+    evaluateJavaScript(script: string): Promise<any>;
+    /**
+     * Add a dump update listener
+     * @param listener Listener function
+     * @returns A remove function that can be called to remove this listener
+     */
+    addDumpUpdateListener(listener: (dump: string, executionDump?: ExecutionDump) => void): () => void;
+    /**
+     * Remove a dump update listener
+     * @param listener The listener function to remove
+     */
+    removeDumpUpdateListener(listener: (dump: string, executionDump?: ExecutionDump) => void): void;
+    /**
+     * Clear all dump update listeners
+     */
+    clearDumpUpdateListeners(): void;
+    destroy(): Promise<void>;
+    recordToReport(title?: string, opt?: {
+        content: string;
+    }): Promise<void>;
+    /**
+     * @deprecated Use {@link Agent.recordToReport} instead.
+     */
+    logScreenshot(title?: string, opt?: {
+        content: string;
+    }): Promise<void>;
+    _unstableLogContent(): {
+        groupName: string;
+        groupDescription: string | undefined;
+        executions: ExecutionDump[];
+    };
+    /**
+     * Freezes the current page context to be reused in subsequent AI operations
+     * This avoids recalculating page context for each operation
+     */
+    freezePageContext(): Promise<void>;
+    /**
+     * Unfreezes the page context, allowing AI operations to calculate context dynamically
+     */
+    unfreezePageContext(): Promise<void>;
+    /**
+     * Process cache configuration and return normalized cache settings
+     */
+    private processCacheConfig;
+    private normalizeFilePaths;
+    private normalizeFileInput;
+    /**
+     * Manually flush cache to file
+     * @param options - Optional configuration
+     * @param options.cleanUnused - If true, removes unused cache records before flushing
+     */
+    flushCache(options?: {
+        cleanUnused?: boolean;
+    }): Promise<void>;
+}
+export declare const createAgent: (interfaceInstance: AbstractInterface, opts?: AgentOpt) => Agent<AbstractInterface>;

package/dist/types/agent/execution-session.d.ts

@@ -0,0 +1,36 @@
+import { type TaskExecutionError, TaskRunner } from '../task-runner';
+import type { ExecutionTaskApply, ExecutionTaskProgressOptions, UIContext } from '../types';
+type ExecutionSessionOptions = ExecutionTaskProgressOptions & {
+    tasks?: ExecutionTaskApply[];
+    onTaskUpdate?: (runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void;
+};
+/**
+ * Thin wrapper around {@link TaskRunner} that represents a single linear execution run.
+ */
+export declare class ExecutionSession {
+    private readonly runner;
+    constructor(name: string, contextProvider: () => Promise<UIContext>, options?: ExecutionSessionOptions);
+    append(tasks: ExecutionTaskApply[] | ExecutionTaskApply, options?: {
+        allowWhenError?: boolean;
+    }): Promise<void>;
+    appendAndRun(tasks: ExecutionTaskApply[] | ExecutionTaskApply, options?: {
+        allowWhenError?: boolean;
+    }): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
+    run(options?: {
+        allowWhenError?: boolean;
+    }): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
+    isInErrorState(): boolean;
+    latestErrorTask(): import("../types").ExecutionTask | null;
+    appendErrorPlan(errorMsg: string): Promise<{
+        output: undefined;
+        runner: TaskRunner;
+    }>;
+    getRunner(): TaskRunner;
+}
+export {};

package/dist/types/agent/index.d.ts

@@ -0,0 +1,9 @@
+export { Agent, createAgent } from './agent';
+export { commonContextParser } from './utils';
+export { getReportFileName, printReportMsg, } from './utils';
+export { extractInsightParam, locateParamStr, paramStr, taskTitleStr, typeStr, } from './ui-utils';
+export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
+export { cacheFileExt } from './task-cache';
+export { TaskExecutor } from './tasks';
+export type { AgentOpt } from '../types';
+export type { AiActOptions } from './agent';

package/dist/types/agent/task-builder.d.ts

@@ -0,0 +1,34 @@
+import type { AbstractInterface } from '../device';
+import type Service from '../service';
+import type { DetailedLocateParam, DeviceAction, ExecutionTaskApply, PlanningAction, PlanningLocateParam } from '../types';
+import type { IModelConfig } from '@godscene/shared/env';
+import type { TaskCache } from './task-cache';
+export declare function locatePlanForLocate(param: string | DetailedLocateParam): PlanningAction<PlanningLocateParam>;
+interface TaskBuilderDeps {
+    interfaceInstance: AbstractInterface;
+    service: Service;
+    taskCache?: TaskCache;
+    actionSpace: DeviceAction[];
+    waitAfterAction?: number;
+}
+interface BuildOptions {
+    cacheable?: boolean;
+    deepLocate?: boolean;
+    abortSignal?: AbortSignal;
+}
+export declare class TaskBuilder {
+    private readonly interface;
+    private readonly service;
+    private readonly taskCache?;
+    private readonly actionSpace;
+    private readonly waitAfterAction?;
+    constructor({ interfaceInstance, service, taskCache, actionSpace, waitAfterAction, }: TaskBuilderDeps);
+    build(plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, options?: BuildOptions): Promise<{
+        tasks: ExecutionTaskApply[];
+    }>;
+    private handleFinishedPlan;
+    private handleLocatePlan;
+    private handleActionPlan;
+    private createLocateTask;
+}
+export {};

package/dist/types/agent/task-cache.d.ts

@@ -0,0 +1,49 @@
+import type { TUserPrompt } from '../ai-model';
+import type { ElementCacheFeature } from '../types';
+export declare const debug: import("@godscene/shared/logger").DebugFunction;
+export interface PlanningCache {
+    type: 'plan';
+    prompt: string;
+    yamlWorkflow: string;
+}
+export interface LocateCache {
+    type: 'locate';
+    prompt: TUserPrompt;
+    cache?: ElementCacheFeature;
+    /** @deprecated kept for backward compatibility */
+    xpaths?: string[];
+}
+export interface MatchCacheResult<T extends PlanningCache | LocateCache> {
+    cacheContent: T;
+    cacheUsable: boolean;
+    updateFn: (cb: (cache: T) => void) => void;
+}
+export type CacheFileContent = {
+    midsceneVersion: string;
+    cacheId: string;
+    caches: Array<PlanningCache | LocateCache>;
+};
+export declare const cacheFileExt = ".cache.yaml";
+export declare class TaskCache {
+    cacheId: string;
+    cacheFilePath?: string;
+    cache: CacheFileContent;
+    isCacheResultUsed: boolean;
+    cacheOriginalLength: number;
+    readOnlyMode: boolean;
+    writeOnlyMode: boolean;
+    private matchedCacheIndices;
+    constructor(cacheId: string, isCacheResultUsed: boolean, cacheFilePath?: string, options?: {
+        readOnly?: boolean;
+        writeOnly?: boolean;
+    });
+    matchCache(prompt: TUserPrompt, type: 'plan' | 'locate'): MatchCacheResult<PlanningCache | LocateCache> | undefined;
+    matchPlanCache(prompt: string): MatchCacheResult<PlanningCache> | undefined;
+    matchLocateCache(prompt: TUserPrompt): MatchCacheResult<LocateCache> | undefined;
+    appendCache(cache: PlanningCache | LocateCache): void;
+    loadCacheFromFile(): CacheFileContent | undefined;
+    flushCacheToFile(options?: {
+        cleanUnused?: boolean;
+    }): void;
+    updateOrAppendCacheRecord(newRecord: PlanningCache | LocateCache, cachedRecord?: MatchCacheResult<PlanningCache | LocateCache>): void;
+}

package/dist/types/agent/tasks.d.ts

@@ -0,0 +1,70 @@
+import { type TMultimodalPrompt, type TUserPrompt } from '../common';
+import type { AbstractInterface } from '../device';
+import type Service from '../service';
+import type { TaskRunner } from '../task-runner';
+import { TaskExecutionError } from '../task-runner';
+import type { DeepThinkOption, DeviceAction, ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam } from '../types';
+import type { IModelConfig } from '@godscene/shared/env';
+import type { TaskCache } from './task-cache';
+export { locatePlanForLocate } from './task-builder';
+interface ExecutionResult<OutputType = any> {
+    output: OutputType;
+    thought?: string;
+    runner: TaskRunner;
+}
+interface TaskExecutorHooks {
+    onTaskUpdate?: (runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void;
+}
+export { TaskExecutionError };
+export declare class TaskExecutor {
+    interface: AbstractInterface;
+    service: Service;
+    taskCache?: TaskCache;
+    private readonly providedActionSpace;
+    private readonly taskBuilder;
+    onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];
+    private readonly hooks?;
+    replanningCycleLimit?: number;
+    waitAfterAction?: number;
+    useDeviceTime?: boolean;
+    get page(): AbstractInterface;
+    constructor(interfaceInstance: AbstractInterface, service: Service, opts: {
+        taskCache?: TaskCache;
+        onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
+        replanningCycleLimit?: number;
+        waitAfterAction?: number;
+        useDeviceTime?: boolean;
+        hooks?: TaskExecutorHooks;
+        actionSpace: DeviceAction[];
+    });
+    private createExecutionSession;
+    private getActionSpace;
+    /**
+     * Get a readable time string. When device time is enabled, use the
+     * device-formatted wall-clock time directly so host timezone formatting does
+     * not reinterpret a device timestamp.
+     * @param format - Optional format string
+     * @returns A formatted time string
+     */
+    private getTimeString;
+    convertPlanToExecutable(plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, options?: {
+        cacheable?: boolean;
+        deepLocate?: boolean;
+        abortSignal?: AbortSignal;
+    }): Promise<{
+        tasks: ExecutionTaskApply[];
+    }>;
+    loadYamlFlowAsPlanning(userInstruction: string, yamlString: string): Promise<{
+        runner: TaskRunner;
+    }>;
+    runPlans(title: string, plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig): Promise<ExecutionResult>;
+    action(userPrompt: string, modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, includeBboxInPlanning: boolean, aiActContext?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, imagesIncludeCount?: number, deepThink?: DeepThinkOption, fileChooserAccept?: string[], deepLocate?: boolean, abortSignal?: AbortSignal): Promise<ExecutionResult<{
+        yamlFlow?: MidsceneYamlFlowItem[];
+        output?: string;
+    } | undefined>>;
+    private runAction;
+    private createTypeQueryTask;
+    createTypeQueryExecution<T>(type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert', demand: ServiceExtractParam, modelConfig: IModelConfig, opt?: ServiceExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<ExecutionResult<T>>;
+    waitFor(assertion: TUserPrompt, opt: PlanningActionParamWaitFor, modelConfig: IModelConfig): Promise<ExecutionResult<void>>;
+}
+export declare function withFileChooser<T>(interfaceInstance: AbstractInterface, fileChooserAccept: string[] | undefined, action: () => Promise<T>): Promise<T>;

package/dist/types/agent/ui-utils.d.ts

@@ -0,0 +1,14 @@
+import type { DetailedLocateParam, ExecutionTask, PullParam, ScrollParam } from '../types';
+export declare function typeStr(task: ExecutionTask): any;
+export declare function locateParamStr(locate?: DetailedLocateParam | string): string;
+export declare function scrollParamStr(scrollParam?: ScrollParam): string;
+export declare function pullParamStr(pullParam?: PullParam): string;
+export declare function extractInsightParam(taskParam: any): {
+    content: string;
+    images?: Array<{
+        name: string;
+        url: string;
+    }>;
+};
+export declare function taskTitleStr(type: 'Tap' | 'Hover' | 'Input' | 'RightClick' | 'KeyboardPress' | 'Scroll' | 'Act' | 'Query' | 'Assert' | 'WaitFor' | 'Locate' | 'Boolean' | 'Number' | 'String', prompt: string): string;
+export declare function paramStr(task: ExecutionTask): string;

package/dist/types/agent/utils.d.ts

@@ -0,0 +1,25 @@
+import type { TMultimodalPrompt, TUserPrompt } from '../common';
+import type { AbstractInterface } from '../device';
+import type { ElementCacheFeature, LocateResultElement, PlanningLocateParam, Rect, UIContext } from '../types';
+import type { TModelFamily } from '@godscene/shared/env';
+import type { TaskCache } from './task-cache';
+export declare function commonContextParser(interfaceInstance: AbstractInterface, _opt: {
+    uploadServerUrl?: string;
+    screenshotShrinkFactor?: number;
+    modelFamily?: TModelFamily;
+}): Promise<UIContext>;
+export declare function getReportFileName(tag?: string): string;
+export declare function printReportMsg(filepath: string): void;
+export declare function ifPlanLocateParamIsBbox(planLocateParam: PlanningLocateParam): boolean;
+export declare function matchElementFromPlan(planLocateParam: PlanningLocateParam): LocateResultElement | undefined;
+export declare function matchElementFromCache(context: {
+    taskCache?: TaskCache;
+    interfaceInstance: AbstractInterface;
+}, cacheEntry: ElementCacheFeature | undefined, cachePrompt: TUserPrompt, cacheable: boolean | undefined): Promise<LocateResultElement | undefined>;
+export declare const getMidsceneVersion: () => string;
+export declare const parsePrompt: (prompt: TUserPrompt) => {
+    textPrompt: string;
+    multimodalPrompt?: TMultimodalPrompt;
+};
+export declare const transformLogicalElementToScreenshot: (element: LocateResultElement, shrunkShotToLogicalRatio: number) => LocateResultElement;
+export declare const transformLogicalRectToScreenshotRect: (rect: Rect, shrunkShotToLogicalRatio: number) => Rect;

package/dist/types/ai-model/auto-glm/actions.d.ts

@@ -0,0 +1,78 @@
+import type { DeviceAction } from '../../device';
+import type { PlanningAction } from '../../types';
+export interface BaseAction {
+    _metadata: string;
+    think?: string;
+}
+export interface TapAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Tap';
+    element: [number, number];
+}
+export interface DoubleTapAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Double Tap';
+    element: [number, number];
+}
+export interface TypeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Type';
+    text: string;
+}
+export interface SwipeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Swipe';
+    start: [number, number];
+    end: [number, number];
+}
+export interface LongPressAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Long Press';
+    element: [number, number];
+}
+export interface LaunchAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Launch';
+    app: string;
+}
+export interface BackAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Back';
+}
+export interface HomeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Home';
+}
+export interface WaitAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Wait';
+    durationMs: number;
+}
+export interface InteractAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Interact';
+}
+export interface CallAPIAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Call_API';
+    instruction: string;
+}
+export interface TakeoverAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Take_over';
+    message: string;
+}
+export interface NoteAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Note';
+    message: string;
+}
+export interface FinishAction extends BaseAction {
+    _metadata: 'finish';
+    message: string;
+}
+export type ParsedAction = TapAction | DoubleTapAction | TypeAction | SwipeAction | LongPressAction | LaunchAction | BackAction | HomeAction | WaitAction | InteractAction | CallAPIAction | TakeoverAction | NoteAction | FinishAction;
+export declare function transformAutoGLMAction(action: ParsedAction, size: {
+    width: number;
+    height: number;
+}, actionSpace?: DeviceAction[]): PlanningAction[];

package/dist/types/ai-model/auto-glm/index.d.ts

@@ -0,0 +1,6 @@
+export { getAutoGLMLocatePrompt, getAutoGLMPlanPrompt } from './prompt';
+export { parseAutoGLMLocateResponse, parseAutoGLMResponse, parseAction, } from './parser';
+export { autoGLMPlanning } from './planning';
+export { transformAutoGLMAction } from './actions';
+export { isAutoGLM, isUITars } from './util';
+export type { ParsedAction } from './actions';

package/dist/types/ai-model/auto-glm/parser.d.ts

@@ -0,0 +1,18 @@
+import type { ParsedAction } from './actions';
+export declare const extractValueAfter: (src: string, key: string) => string;
+export declare function parseAction(response: {
+    think: string;
+    content: string;
+}): ParsedAction;
+export declare function parseAutoGLMResponse(content: string): {
+    think: string;
+    content: string;
+};
+export declare function parseAutoGLMLocateResponse(rawResponse: string): {
+    think: string;
+    coordinates: {
+        x: number;
+        y: number;
+    } | null;
+    error?: string;
+};

package/dist/types/ai-model/auto-glm/planning.d.ts

@@ -0,0 +1,12 @@
+import type { DeviceAction } from '../../device';
+import type { PlanningAIResponse, UIContext } from '../../types';
+import type { IModelConfig } from '@godscene/shared/env';
+import type { ConversationHistory } from '../conversation-history';
+export declare function autoGLMPlanning(userInstruction: string, options: {
+    conversationHistory: ConversationHistory;
+    context: UIContext;
+    modelConfig: IModelConfig;
+    actionContext?: string;
+    actionSpace?: DeviceAction[];
+    abortSignal?: AbortSignal;
+}): Promise<PlanningAIResponse>;

package/dist/types/ai-model/auto-glm/prompt.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Auto-GLM Prompt Templates
+ *
+ * Portions of this file are derived from Open-AutoGLM
+ * Copyright (c) 2024 zai-org
+ * Licensed under the Apache License, Version 2.0
+ *
+ * Source: https://github.com/zai-org/Open-AutoGLM
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications:
+ * - Adapted prompts for RPA智能助手 integration
+ */
+import type { TModelFamily } from '@godscene/shared/env';
+export declare const getAutoGLMPlanPrompt: (modelFamily: TModelFamily | undefined) => string;
+export declare const getAutoGLMLocatePrompt: (modelFamily: TModelFamily | undefined) => string;

package/dist/types/ai-model/auto-glm/util.d.ts

@@ -0,0 +1,13 @@
+import type { TModelFamily } from '@godscene/shared/env';
+/**
+ * Check if the modelFamily is auto-glm or auto-glm-multilingual
+ * @param modelFamily The model family to check
+ * @returns true if modelFamily is auto-glm or auto-glm-multilingual
+ */
+export declare function isAutoGLM(modelFamily: TModelFamily | undefined): boolean;
+/**
+ * Check if the modelFamily is a UI-TARS variant
+ * @param modelFamily The model family to check
+ * @returns true if modelFamily is any UI-TARS variant
+ */
+export declare function isUITars(modelFamily: TModelFamily | undefined): boolean;

package/dist/types/ai-model/connectivity.d.ts

@@ -0,0 +1,20 @@
+import type { IModelConfig, TIntent } from '@godscene/shared/env';
+export interface ConnectivityCheckResultItem {
+    name: 'text' | 'vision' | 'aiLocate';
+    intent: TIntent;
+    modelName: string;
+    modelFamily?: string;
+    passed: boolean;
+    durationMs: number;
+    message: string;
+}
+export interface ConnectivityTestResult {
+    passed: boolean;
+    checks: ConnectivityCheckResultItem[];
+}
+export interface ConnectivityTestConfig {
+    defaultModelConfig: IModelConfig;
+    planningModelConfig: IModelConfig;
+    insightModelConfig: IModelConfig;
+}
+export declare function runConnectivityTest(config: ConnectivityTestConfig): Promise<ConnectivityTestResult>;