agi 0.3.0 → 0.4.0

package/dist/index.d.ts

@@ -1,3 +1,5 @@
+import { EventEmitter } from 'events';
+
 /**
  * Result types for task execution
  */
@@ -98,8 +100,9 @@ declare class Screenshot {
  */
 
 type SnapshotMode = 'none' | 'memory' | 'filesystem';
+type AgentSessionType = 'managed-cdp' | 'external-cdp' | 'desktop';
 type SessionStatus = 'ready' | 'running' | 'waiting_for_input' | 'paused' | 'finished' | 'error';
-type EventType = 'step' | 'thought' | 'question' | 'done' | 'error' | 'log' | 'paused' | 'resumed' | 'heartbeat' | 'user';
+type EventType$1 = 'step' | 'thought' | 'question' | 'done' | 'error' | 'log' | 'paused' | 'resumed' | 'heartbeat' | 'user';
 type MessageType = 'THOUGHT' | 'QUESTION' | 'USER' | 'DONE' | 'ERROR' | 'LOG';
 interface AGIClientOptions {
     /** API key for authentication */
@@ -120,6 +123,10 @@ interface CreateSessionOptions {
     maxSteps?: number;
     /** Environment UUID to restore from */
     restoreFromEnvironmentId?: string;
+    /** Agent session type */
+    agentSessionType?: AgentSessionType;
+    /** External CDP WebSocket URL (required for external-cdp session type) */
+    cdpUrl?: string;
 }
 interface SessionResponse {
     /** Session UUID */
@@ -138,6 +145,8 @@ interface SessionResponse {
     environmentId?: string;
     /** Task goal */
     goal?: string;
+    /** Agent session type */
+    agentSessionType?: string;
 }
 interface ExecuteStatusResponse {
     /** Current execution status */
@@ -167,7 +176,7 @@ interface SSEEvent {
     /** Event ID */
     id?: string;
     /** Event type */
-    event: EventType;
+    event: EventType$1;
     /** Event data */
     data: Record<string, unknown>;
 }
@@ -215,12 +224,71 @@ interface RunTaskOptions {
 }
 interface StreamOptions {
     /** Event types to filter */
-    eventTypes?: EventType[];
+    eventTypes?: EventType$1[];
     /** Sanitize event data */
     sanitize?: boolean;
     /** Include historical events */
     includeHistory?: boolean;
 }
+type DesktopActionType = 'click' | 'double_click' | 'triple_click' | 'right_click' | 'hover' | 'type' | 'key' | 'scroll' | 'drag' | 'wait' | 'finish' | 'fail' | 'confirm' | 'ask_question';
+interface DesktopAction {
+    /** Action type */
+    type: DesktopActionType | string;
+    /** X coordinate (for click, hover, scroll) */
+    x?: number;
+    /** Y coordinate (for click, hover, scroll) */
+    y?: number;
+    /** Text to type (for type action) */
+    text?: string;
+    /** Content to type (alias for text) */
+    content?: string;
+    /** Key or key combination (for key action) */
+    key?: string;
+    /** Scroll direction (for scroll action) */
+    direction?: 'up' | 'down' | 'left' | 'right';
+    /** Scroll amount in lines (for scroll action) */
+    amount?: number;
+    /** Wait duration in seconds (for wait action) */
+    duration?: number;
+    /** Start X coordinate (for drag action) */
+    start_x?: number;
+    /** Start Y coordinate (for drag action) */
+    start_y?: number;
+    /** End X coordinate (for drag action) */
+    end_x?: number;
+    /** End Y coordinate (for drag action) */
+    end_y?: number;
+    /** Summary message (for finish action) */
+    summary?: string;
+    /** Reason message (for fail action) */
+    reason?: string;
+    /** Question to ask (for ask_question action) */
+    question?: string;
+    /** Additional action properties */
+    [key: string]: unknown;
+}
+interface StepDesktopRequest {
+    /** Base64-encoded screenshot (full resolution). Viewport dimensions are extracted from the image. */
+    screenshot: string;
+    /** Optional user message (e.g., initial goal or follow-up instruction) */
+    message?: string;
+}
+interface StepDesktopResponse {
+    /** Actions to execute (flat format) */
+    actions: DesktopAction[];
+    /** Model reasoning */
+    thinking?: string;
+    /** Whether the task is complete */
+    finished: boolean;
+    /** Question for user, if any */
+    askUser?: string;
+    /** Current step number */
+    step: number;
+}
+interface ModelsResponse {
+    /** Available agent models */
+    models: string[];
+}
 
 /**
  * Session context manager for high-level API
@@ -431,6 +499,10 @@ declare class HTTPClient {
      * Make an HTTP request with retries and error handling
      */
     request<T>(method: string, path: string, options?: RequestOptions): Promise<T>;
+    /**
+     * Make an HTTP request to an absolute URL with retries and error handling
+     */
+    requestUrl<T>(method: string, url: string, options?: RequestOptions): Promise<T>;
     /**
      * Stream Server-Sent Events from an endpoint
      */
@@ -455,16 +527,24 @@ declare class SessionsResource {
     /**
      * Create a new agent session
      *
-     * @param agentName - Agent model to use (e.g., "agi-0", "agi-0-fast", "agi-1")
+     * @param agentName - Agent model to use (e.g., "agi-0", "agi-2-claude")
      * @param options - Session creation options
-     * @returns SessionResponse with session_id, vnc_url, status, etc.
+     * @returns SessionResponse with sessionId, vncUrl, agentUrl, status, etc.
      *
      * @example
      * ```typescript
+     * // Standard browser session
      * const session = await client.sessions.create('agi-0', {
      *   webhookUrl: 'https://yourapp.com/webhook',
      *   maxSteps: 200
      * });
+     *
+     * // Desktop session (client-managed)
+     * const session = await client.sessions.create('agi-2-claude', {
+     *   agentSessionType: 'desktop',
+     *   goal: 'Open calculator and compute 2+2'
+     * });
+     * console.log(session.agentUrl); // Use with client.desktop.step()
      * ```
      */
     create(agentName?: string, options?: {
@@ -472,6 +552,8 @@ declare class SessionsResource {
         goal?: string;
         maxSteps?: number;
         restoreFromEnvironmentId?: string;
+        agentSessionType?: string;
+        cdpUrl?: string;
     }): Promise<SessionResponse>;
     /**
      * List all sessions for the authenticated user
@@ -664,6 +746,66 @@ declare class SessionsResource {
      * ```
      */
     screenshot(sessionId: string): Promise<ScreenshotResponse>;
+    /**
+     * Execute a single step for client-driven sessions (desktop mode).
+     *
+     * In desktop mode (agentSessionType="desktop"), the client manages the
+     * execution loop. This method sends a screenshot to the agent and receives
+     * actions to execute locally.
+     *
+     * @param agentUrl - Agent service URL from session.agentUrl
+     * @param sessionId - Session ID (required for routing in shared sandbox)
+     * @param screenshot - Base64-encoded screenshot (full resolution, JPEG or PNG)
+     * @param message - Optional user message (goal on first call, or follow-up instruction)
+     * @returns StepDesktopResponse with actions, thinking, finished, askUser, and step
+     *
+     * @example
+     * ```typescript
+     * // Create a desktop session
+     * const session = await client.sessions.create('agi-2-claude', {
+     *   agentSessionType: 'desktop',
+     *   goal: 'Open calculator and compute 2+2'
+     * });
+     *
+     * // Client-managed loop
+     * let finished = false;
+     * while (!finished) {
+     *   const screenshot = captureScreenshot(); // Client captures
+     *   const result = await client.sessions.step(
+     *     session.agentUrl!,
+     *     session.sessionId,
+     *     screenshot
+     *   );
+     *   executeActions(result.actions); // Client executes
+     *   finished = result.finished;
+     *   if (result.askUser) {
+     *     const answer = await promptUser(result.askUser);
+     *     // Send answer in next step
+     *   }
+     * }
+     * ```
+     */
+    step(agentUrl: string, sessionId: string, screenshot: string, message?: string): Promise<StepDesktopResponse>;
+    /**
+     * List available agent models.
+     *
+     * @param filter - Optional filter: "cdp" for browser agents, "desktop" for
+     *                 desktop agents
+     * @returns ModelsResponse with list of available model names
+     *
+     * @example
+     * ```typescript
+     * // List all models
+     * const models = await client.sessions.listModels();
+     * console.log(models.models);
+     *
+     * // List only desktop-compatible models
+     * const desktopModels = await client.sessions.listModels('desktop');
+     * console.log(desktopModels.models);
+     * // ['agi-2-claude', 'agi-2-qwen']
+     * ```
+     */
+    listModels(filter?: string): Promise<ModelsResponse>;
 }
 
 /**
@@ -767,6 +909,179 @@ declare class AGIClient {
     }): SessionContext;
 }
 
+/**
+ * Async event loop manager for client-driven sessions.
+ */
+
+/**
+ * State of the agent execution loop.
+ */
+type LoopState = 'idle' | 'running' | 'paused' | 'stopped' | 'finished';
+/**
+ * Callback type for capturing screenshots.
+ * Should return a base64-encoded screenshot.
+ */
+type CaptureScreenshotFn = () => Promise<string>;
+/**
+ * Callback type for executing actions.
+ */
+type ExecuteActionsFn = (actions: DesktopAction[]) => Promise<void>;
+/**
+ * Callback type for handling thinking output.
+ */
+type OnThinkingFn = (thinking: string) => void;
+/**
+ * Callback type for handling user questions.
+ * Should return the user's answer.
+ */
+type OnAskUserFn = (question: string) => Promise<string>;
+/**
+ * Callback type for step completion.
+ */
+type OnStepFn = (step: number, response: StepDesktopResponse) => void;
+/**
+ * Options for creating an AgentLoop.
+ */
+interface AgentLoopOptions {
+    /** AGIClient instance */
+    client: AGIClient;
+    /** Agent service URL from session.agentUrl */
+    agentUrl: string;
+    /** Session ID (required for routing in shared sandbox) */
+    sessionId: string;
+    /** Async callback that returns base64-encoded screenshot */
+    captureScreenshot: CaptureScreenshotFn;
+    /** Async callback that executes a list of actions */
+    executeActions: ExecuteActionsFn;
+    /** Optional callback for agent thinking/reasoning output */
+    onThinking?: OnThinkingFn;
+    /** Optional async callback to handle user questions. If not provided and agent asks a question, the loop will stop. */
+    onAskUser?: OnAskUserFn;
+    /** Optional callback called after each step with step number and full response */
+    onStep?: OnStepFn;
+    /** Optional delay in milliseconds between steps (default: 0) */
+    stepDelay?: number;
+}
+/**
+ * Async event loop manager for client-driven sessions.
+ *
+ * This class manages the execution loop for agent sessions where the client
+ * is responsible for capturing screenshots and executing actions (e.g., desktop,
+ * mobile, or any client-driven session type). It provides start/pause/resume/stop
+ * control over the loop.
+ *
+ * The loop:
+ * 1. Captures a screenshot using the provided callback
+ * 2. Sends it to the agent via step()
+ * 3. Executes returned actions using the provided callback
+ * 4. Repeats until finished or stopped
+ *
+ * @example
+ * ```typescript
+ * import { AGIClient, AgentLoop } from 'agi-sdk';
+ *
+ * const client = new AGIClient({ apiKey: '...' });
+ *
+ * // Create client-driven session
+ * const session = await client.sessions.create('agi-2-claude', {
+ *   agentSessionType: 'desktop',
+ *   goal: 'Open calculator and compute 2+2'
+ * });
+ *
+ * // Create and run loop
+ * const loop = new AgentLoop({
+ *   client,
+ *   agentUrl: session.agentUrl!,
+ *   captureScreenshot: async () => {
+ *     // Return base64-encoded screenshot
+ *     return '...';
+ *   },
+ *   executeActions: async (actions) => {
+ *     for (const action of actions) {
+ *       console.log('Executing:', action);
+ *     }
+ *   },
+ *   onThinking: (t) => console.log('Thinking:', t),
+ * });
+ *
+ * const result = await loop.start();
+ * console.log('Finished:', result.finished);
+ * ```
+ *
+ * @example Pause/resume control
+ * ```typescript
+ * // Start loop without awaiting
+ * const promise = loop.start();
+ *
+ * // Pause after 5 seconds
+ * setTimeout(() => loop.pause(), 5000);
+ *
+ * // Resume after user input
+ * process.stdin.once('data', () => loop.resume());
+ *
+ * // Wait for completion
+ * const result = await promise;
+ * ```
+ */
+declare class AgentLoop {
+    private readonly client;
+    private readonly agentUrl;
+    private readonly sessionId;
+    private readonly captureScreenshot;
+    private readonly executeActions;
+    private readonly onThinking?;
+    private readonly onAskUser?;
+    private readonly onStep?;
+    private readonly stepDelay;
+    private _state;
+    private _lastResult;
+    private _currentStep;
+    private pauseResolve;
+    private pausePromise;
+    constructor(options: AgentLoopOptions);
+    /** Current state of the loop. */
+    get state(): LoopState;
+    /** Current step number. */
+    get currentStep(): number;
+    /** Last step result, if any. */
+    get lastResult(): StepDesktopResponse | null;
+    /**
+     * Start the execution loop.
+     *
+     * Runs the loop until the task is finished, stopped, or an unhandled
+     * ask_user question is encountered.
+     *
+     * @param message - Optional initial message (goal or instruction).
+     *   Usually not needed if goal was set during session creation.
+     * @returns The final StepDesktopResponse
+     * @throws Error if loop is already running
+     */
+    start(message?: string): Promise<StepDesktopResponse>;
+    /**
+     * Pause the execution loop.
+     *
+     * The loop will complete the current step before pausing.
+     * Call resume() to continue.
+     */
+    pause(): void;
+    /**
+     * Resume a paused loop.
+     */
+    resume(): void;
+    /**
+     * Stop the execution loop.
+     *
+     * The loop will complete the current step before stopping.
+     */
+    stop(): void;
+    /** Check if the loop is currently running. */
+    isRunning(): boolean;
+    /** Check if the loop is currently paused. */
+    isPaused(): boolean;
+    /** Check if the loop has finished successfully. */
+    isFinished(): boolean;
+}
+
 /**
  * Error classes for AGI SDK
  */
@@ -821,4 +1136,292 @@ declare class APIError extends AGIError {
     constructor(message: string, statusCode?: number, response?: unknown);
 }
 
-export { AGIClient, type AGIClientOptions, AGIError, APIError, AgentExecutionError, AuthenticationError, type CreateSessionOptions, type DeleteResponse, type EventType, type ExecuteStatusResponse, type MessageOptions, type MessageResponse, type MessageType, type MessagesResponse, type NavigateResponse, NotFoundError, PermissionError, RateLimitError, type RunTaskOptions, type SSEEvent, Screenshot, type ScreenshotResponse, SessionContext, type SessionResponse, type SessionStatus, SessionsResource, type SnapshotMode, type StreamOptions, type SuccessResponse, type TaskMetadata, type TaskResult, ValidationError };
+/**
+ * Protocol types for driver communication.
+ *
+ * The driver communicates via JSON lines:
+ * - Events are emitted on stdout (driver -> SDK)
+ * - Commands are sent on stdin (SDK -> driver)
+ */
+type EventType = 'ready' | 'state_change' | 'thinking' | 'action' | 'confirm' | 'ask_question' | 'finished' | 'error' | 'screenshot_captured';
+type CommandType = 'start' | 'screenshot' | 'pause' | 'resume' | 'stop' | 'confirm' | 'answer';
+type DriverState = 'idle' | 'running' | 'paused' | 'waiting_confirmation' | 'waiting_answer' | 'finished' | 'stopped' | 'error';
+interface BaseEvent {
+    event: EventType;
+    step: number;
+}
+interface ReadyEvent extends BaseEvent {
+    event: 'ready';
+    version: string;
+    protocol: string;
+}
+interface StateChangeEvent extends BaseEvent {
+    event: 'state_change';
+    state: DriverState;
+}
+interface ThinkingEvent extends BaseEvent {
+    event: 'thinking';
+    text: string;
+}
+interface ActionEvent extends BaseEvent {
+    event: 'action';
+    action: DriverAction;
+}
+interface ConfirmEvent extends BaseEvent {
+    event: 'confirm';
+    action: DriverAction;
+    reason: string;
+}
+interface AskQuestionEvent extends BaseEvent {
+    event: 'ask_question';
+    question: string;
+    question_id: string;
+}
+interface FinishedEvent extends BaseEvent {
+    event: 'finished';
+    reason: string;
+    summary: string;
+    success: boolean;
+}
+interface ErrorEvent extends BaseEvent {
+    event: 'error';
+    message: string;
+    code: string;
+    recoverable: boolean;
+}
+/**
+ * Emitted in local mode when the driver captures a screenshot.
+ * Lightweight notification (no image data).
+ */
+interface ScreenshotCapturedEvent extends BaseEvent {
+    event: 'screenshot_captured';
+    width: number;
+    height: number;
+}
+type DriverEvent = ReadyEvent | StateChangeEvent | ThinkingEvent | ActionEvent | ConfirmEvent | AskQuestionEvent | FinishedEvent | ErrorEvent | ScreenshotCapturedEvent;
+interface DriverAction {
+    type: string;
+    x?: number;
+    y?: number;
+    [key: string]: unknown;
+}
+interface BaseCommand {
+    command: CommandType;
+}
+interface StartCommand extends BaseCommand {
+    command: 'start';
+    session_id: string;
+    goal: string;
+    screenshot: string;
+    screen_width: number;
+    screen_height: number;
+    platform: 'desktop' | 'android';
+    model: string;
+    /** "local" for autonomous mode, "" for legacy SDK-driven mode */
+    mode?: string;
+}
+interface ScreenshotCommand extends BaseCommand {
+    command: 'screenshot';
+    data: string;
+    screen_width: number;
+    screen_height: number;
+}
+interface PauseCommand extends BaseCommand {
+    command: 'pause';
+}
+interface ResumeCommand extends BaseCommand {
+    command: 'resume';
+}
+interface StopCommand extends BaseCommand {
+    command: 'stop';
+    reason?: string;
+}
+interface ConfirmResponseCommand extends BaseCommand {
+    command: 'confirm';
+    approved: boolean;
+    message?: string;
+}
+interface AnswerCommand extends BaseCommand {
+    command: 'answer';
+    text: string;
+    question_id?: string;
+}
+type DriverCommand = StartCommand | ScreenshotCommand | PauseCommand | ResumeCommand | StopCommand | ConfirmResponseCommand | AnswerCommand;
+
+/**
+ * AgentDriver - Spawns and manages the agi-driver binary.
+ *
+ * The driver communicates via JSON lines over stdin/stdout and provides
+ * an event-based interface for agent control.
+ */
+
+/**
+ * Options for creating an AgentDriver.
+ */
+interface DriverOptions {
+    /** Path to the agi-driver binary. If not provided, will be auto-detected. */
+    binaryPath?: string;
+    /** Model to use (default: 'claude-sonnet') */
+    model?: string;
+    /** Platform type (default: 'desktop') */
+    platform?: 'desktop' | 'android';
+    /** "local" for autonomous mode, "" for legacy SDK-driven mode */
+    mode?: string;
+    /** Environment variables to pass to the driver process */
+    env?: Record<string, string>;
+}
+/**
+ * Result from running the driver.
+ */
+interface DriverResult {
+    /** Whether the task completed successfully */
+    success: boolean;
+    /** Reason for completion */
+    reason: string;
+    /** Summary of what was accomplished */
+    summary: string;
+    /** Final step number */
+    step: number;
+}
+/**
+ * AgentDriver manages the lifecycle of the agi-driver binary.
+ *
+ * @example
+ * ```typescript
+ * const driver = new AgentDriver();
+ *
+ * driver.on('action', async (action) => {
+ *   // Execute the action on the local machine
+ *   await executeAction(action);
+ *   // Send next screenshot
+ *   driver.sendScreenshot(captureScreenshot());
+ * });
+ *
+ * driver.on('thinking', (text) => {
+ *   console.log('Agent thinking:', text);
+ * });
+ *
+ * driver.on('confirm', async (reason) => {
+ *   // Ask user for confirmation
+ *   return await askUser(reason);
+ * });
+ *
+ * const result = await driver.start('Open calculator and compute 2+2');
+ * console.log('Done:', result.summary);
+ * ```
+ */
+declare class AgentDriver extends EventEmitter {
+    private readonly binaryPath;
+    private readonly pythonFallback;
+    private readonly model;
+    private readonly platform;
+    private readonly mode;
+    private readonly env;
+    private process;
+    private readline;
+    private state;
+    private step;
+    private sessionId;
+    private screenWidth;
+    private screenHeight;
+    private resolveStart;
+    private rejectStart;
+    private pendingConfirm;
+    private pendingAnswer;
+    constructor(options?: DriverOptions);
+    /**
+     * Get the current state of the driver.
+     */
+    get currentState(): DriverState;
+    /**
+     * Get the current step number.
+     */
+    get currentStep(): number;
+    /**
+     * Check if the driver is running.
+     */
+    get isRunning(): boolean;
+    /**
+     * Check if the driver is waiting for user input.
+     */
+    get isWaiting(): boolean;
+    /**
+     * Start the agent with a goal.
+     *
+     * @param goal - The task for the agent to accomplish
+     * @param screenshot - Initial screenshot (base64-encoded). Not needed in local mode.
+     * @param screenWidth - Screen width in pixels. Not needed in local mode.
+     * @param screenHeight - Screen height in pixels. Not needed in local mode.
+     * @param mode - Override the mode set in DriverOptions.
+     * @returns Promise that resolves when the agent finishes
+     */
+    start(goal: string, screenshot?: string, screenWidth?: number, screenHeight?: number, mode?: string): Promise<DriverResult>;
+    /**
+     * Send a new screenshot to the driver.
+     *
+     * @param screenshot - Base64-encoded screenshot
+     * @param screenWidth - Screen width in pixels
+     * @param screenHeight - Screen height in pixels
+     */
+    sendScreenshot(screenshot: string, screenWidth?: number, screenHeight?: number): void;
+    /**
+     * Pause the driver.
+     */
+    pause(): void;
+    /**
+     * Resume the driver.
+     */
+    resume(): void;
+    /**
+     * Stop the driver.
+     *
+     * @param reason - Reason for stopping
+     */
+    stop(reason?: string): Promise<void>;
+    /**
+     * Respond to a confirmation request.
+     *
+     * @param approved - Whether the action is approved
+     * @param message - Optional message to send with the response
+     */
+    respondConfirm(approved: boolean, message?: string): void;
+    /**
+     * Respond to a question.
+     *
+     * @param text - The answer text
+     * @param questionId - Optional question ID
+     */
+    respondAnswer(text: string, questionId?: string): void;
+    private sendCommand;
+    private handleLine;
+    private handleFinished;
+    private handleError;
+    private cleanup;
+}
+
+/**
+ * Binary locator for the agi-driver.
+ *
+ * Finds the platform-specific binary bundled with the package
+ * or falls back to a global installation.
+ */
+/**
+ * Platform identifier for binary selection.
+ */
+type PlatformId = 'darwin-arm64' | 'darwin-x64' | 'linux-x64' | 'win32-x64';
+/**
+ * Get the current platform identifier.
+ */
+declare function getPlatformId(): PlatformId;
+/**
+ * Find the agi-driver binary path.
+ *
+ * @throws Error if the binary cannot be found
+ */
+declare function findBinaryPath(): string;
+/**
+ * Check if the binary is available.
+ */
+declare function isBinaryAvailable(): boolean;
+
+export { AGIClient, type AGIClientOptions, AGIError, APIError, AgentDriver, AgentExecutionError, AgentLoop, type AgentLoopOptions, type AgentSessionType, AuthenticationError, type CaptureScreenshotFn, type CreateSessionOptions, type DeleteResponse, type DesktopAction, type DesktopActionType, type DriverAction, type DriverCommand, type DriverEvent, type DriverOptions, type DriverResult, type DriverState, type EventType$1 as EventType, type ExecuteActionsFn, type ExecuteStatusResponse, type LoopState, type MessageOptions, type MessageResponse, type MessageType, type MessagesResponse, type ModelsResponse, type NavigateResponse, NotFoundError, type OnAskUserFn, type OnStepFn, type OnThinkingFn, PermissionError, type PlatformId, RateLimitError, type RunTaskOptions, type SSEEvent, Screenshot, type ScreenshotResponse, SessionContext, type SessionResponse, type SessionStatus, SessionsResource, type SnapshotMode, type StepDesktopRequest, type StepDesktopResponse, type StreamOptions, type SuccessResponse, type TaskMetadata, type TaskResult, ValidationError, findBinaryPath, getPlatformId, isBinaryAvailable };