btcp-browser-agent 0.1.0 → 0.1.1

package/packages/core/dist/types.d.ts

@@ -0,0 +1,396 @@
+/**
+ * @btcp/core - Type definitions
+ *
+ * Core types for DOM automation commands and responses.
+ */
+export type CoreAction = 'click' | 'dblclick' | 'type' | 'fill' | 'clear' | 'check' | 'uncheck' | 'select' | 'focus' | 'blur' | 'hover' | 'scroll' | 'scrollIntoView' | 'snapshot' | 'querySelector' | 'querySelectorAll' | 'getText' | 'getAttribute' | 'getProperty' | 'getBoundingBox' | 'isVisible' | 'isEnabled' | 'isChecked' | 'press' | 'keyDown' | 'keyUp' | 'wait' | 'evaluate' | 'validateElement' | 'validateRefs' | 'highlight' | 'clearHighlight';
+export interface BaseCommand {
+    id: string;
+    action: CoreAction;
+}
+export type Selector = string;
+export interface ClickCommand extends BaseCommand {
+    action: 'click';
+    selector: Selector;
+    button?: 'left' | 'right' | 'middle';
+    clickCount?: number;
+    modifiers?: Modifier[];
+}
+export interface DblClickCommand extends BaseCommand {
+    action: 'dblclick';
+    selector: Selector;
+}
+export interface TypeCommand extends BaseCommand {
+    action: 'type';
+    selector: Selector;
+    text: string;
+    delay?: number;
+    clear?: boolean;
+}
+export interface FillCommand extends BaseCommand {
+    action: 'fill';
+    selector: Selector;
+    value: string;
+}
+export interface ClearCommand extends BaseCommand {
+    action: 'clear';
+    selector: Selector;
+}
+export interface CheckCommand extends BaseCommand {
+    action: 'check';
+    selector: Selector;
+}
+export interface UncheckCommand extends BaseCommand {
+    action: 'uncheck';
+    selector: Selector;
+}
+export interface SelectCommand extends BaseCommand {
+    action: 'select';
+    selector: Selector;
+    values: string | string[];
+}
+export interface FocusCommand extends BaseCommand {
+    action: 'focus';
+    selector: Selector;
+}
+export interface BlurCommand extends BaseCommand {
+    action: 'blur';
+    selector: Selector;
+}
+export interface HoverCommand extends BaseCommand {
+    action: 'hover';
+    selector: Selector;
+}
+export interface ScrollCommand extends BaseCommand {
+    action: 'scroll';
+    selector?: Selector;
+    x?: number;
+    y?: number;
+    direction?: 'up' | 'down' | 'left' | 'right';
+    amount?: number;
+}
+export interface ScrollIntoViewCommand extends BaseCommand {
+    action: 'scrollIntoView';
+    selector: Selector;
+    block?: 'start' | 'center' | 'end' | 'nearest';
+}
+/**
+ * Grep options for filtering snapshot output (mirrors Unix grep flags)
+ */
+export interface GrepOptions {
+    /** Pattern to search for (regex by default) */
+    pattern: string;
+    /** Case-insensitive matching (grep -i) */
+    ignoreCase?: boolean;
+    /** Invert match - return non-matching lines (grep -v) */
+    invert?: boolean;
+    /** Treat pattern as fixed string, not regex (grep -F) */
+    fixedStrings?: boolean;
+}
+export interface SnapshotCommand extends BaseCommand {
+    action: 'snapshot';
+    selector?: Selector;
+    maxDepth?: number;
+    includeHidden?: boolean;
+    interactive?: boolean;
+    compact?: boolean;
+    minDepth?: number;
+    samplingStrategy?: 'importance' | 'balanced' | 'depth-first';
+    contentPreview?: boolean;
+    landmarks?: boolean;
+    incremental?: boolean;
+    baseSnapshot?: SnapshotData;
+    all?: boolean;
+    format?: 'tree' | 'html';
+    /** Filter output lines - simple string or full grep options */
+    grep?: string | GrepOptions;
+}
+export interface QuerySelectorCommand extends BaseCommand {
+    action: 'querySelector';
+    selector: Selector;
+}
+export interface QuerySelectorAllCommand extends BaseCommand {
+    action: 'querySelectorAll';
+    selector: Selector;
+}
+export interface GetTextCommand extends BaseCommand {
+    action: 'getText';
+    selector: Selector;
+}
+export interface GetAttributeCommand extends BaseCommand {
+    action: 'getAttribute';
+    selector: Selector;
+    attribute: string;
+}
+export interface GetPropertyCommand extends BaseCommand {
+    action: 'getProperty';
+    selector: Selector;
+    property: string;
+}
+export interface GetBoundingBoxCommand extends BaseCommand {
+    action: 'getBoundingBox';
+    selector: Selector;
+}
+export interface IsVisibleCommand extends BaseCommand {
+    action: 'isVisible';
+    selector: Selector;
+}
+export interface IsEnabledCommand extends BaseCommand {
+    action: 'isEnabled';
+    selector: Selector;
+}
+export interface IsCheckedCommand extends BaseCommand {
+    action: 'isChecked';
+    selector: Selector;
+}
+export interface PressCommand extends BaseCommand {
+    action: 'press';
+    key: string;
+    selector?: Selector;
+    modifiers?: Modifier[];
+}
+export interface KeyDownCommand extends BaseCommand {
+    action: 'keyDown';
+    key: string;
+}
+export interface KeyUpCommand extends BaseCommand {
+    action: 'keyUp';
+    key: string;
+}
+export interface WaitCommand extends BaseCommand {
+    action: 'wait';
+    selector?: Selector;
+    state?: 'visible' | 'hidden' | 'attached' | 'detached';
+    timeout?: number;
+}
+export interface EvaluateCommand extends BaseCommand {
+    action: 'evaluate';
+    script: string;
+    args?: unknown[];
+}
+/**
+ * Validate element capabilities before attempting an action
+ *
+ * Allows AI agents to check element compatibility and get actionable feedback
+ * before executing commands that might fail.
+ *
+ * @example Pre-validate before typing
+ * ```typescript
+ * const validation = await agent.execute({
+ *   id: 'v1',
+ *   action: 'validateElement',
+ *   selector: '#username',
+ *   capabilities: ['editable']
+ * });
+ *
+ * if (validation.data.compatible) {
+ *   await agent.execute({
+ *     id: 'a1',
+ *     action: 'type',
+ *     selector: '#username',
+ *     text: 'user@example.com'
+ *   });
+ * } else {
+ *   console.log(validation.data.suggestion);
+ * }
+ * ```
+ */
+export interface ValidateElementCommand extends BaseCommand {
+    action: 'validateElement';
+    /** Element selector to validate */
+    selector: Selector;
+    /** Expected element type (optional) */
+    expectedType?: 'input' | 'textarea' | 'button' | 'link' | 'select';
+    /** Required capabilities (optional) */
+    capabilities?: Array<'clickable' | 'editable' | 'checkable' | 'hoverable'>;
+}
+/**
+ * Validate that refs are still valid
+ *
+ * Checks if refs from a previous snapshot are still valid or have expired.
+ * Helps AI agents avoid using stale refs.
+ *
+ * @example Check ref validity
+ * ```typescript
+ * const validation = await agent.execute({
+ *   id: 'v1',
+ *   action: 'validateRefs',
+ *   refs: ['@ref:0', '@ref:1', '@ref:2']
+ * });
+ *
+ * // Use only valid refs
+ * for (const ref of validation.data.valid) {
+ *   await agent.execute({ id: '...', action: 'click', selector: ref });
+ * }
+ *
+ * // Handle invalid refs
+ * if (validation.data.invalid.length > 0) {
+ *   // Take new snapshot to get fresh refs
+ *   await agent.execute({ id: '...', action: 'snapshot' });
+ * }
+ * ```
+ */
+export interface ValidateRefsCommand extends BaseCommand {
+    action: 'validateRefs';
+    /** List of refs to validate */
+    refs: string[];
+}
+/**
+ * Display visual overlay labels for interactive elements
+ *
+ * Shows reference numbers (@ref:0, @ref:1, etc.) as overlay labels
+ * positioned near each interactive element from the last snapshot.
+ * Labels persist until explicitly cleared.
+ *
+ * @example Highlight elements after snapshot
+ * ```typescript
+ * // Take a snapshot first
+ * await agent.execute({ id: 's1', action: 'snapshot' });
+ *
+ * // Show visual highlights
+ * await agent.execute({ id: 'h1', action: 'highlight' });
+ *
+ * // Labels now visible on page with @ref:0, @ref:1, etc.
+ * // Use the refs to interact with elements
+ * await agent.execute({ id: 'c1', action: 'click', selector: '@ref:5' });
+ *
+ * // Clear highlights when done
+ * await agent.execute({ id: 'ch1', action: 'clearHighlight' });
+ * ```
+ */
+export interface HighlightCommand extends BaseCommand {
+    action: 'highlight';
+}
+/**
+ * Remove visual overlay labels
+ *
+ * Clears all highlight overlays from the page.
+ *
+ * @example Clear highlights
+ * ```typescript
+ * await agent.execute({ id: 'ch1', action: 'clearHighlight' });
+ * ```
+ */
+export interface ClearHighlightCommand extends BaseCommand {
+    action: 'clearHighlight';
+}
+export type Modifier = 'Alt' | 'Control' | 'Meta' | 'Shift';
+export type Command = ClickCommand | DblClickCommand | TypeCommand | FillCommand | ClearCommand | CheckCommand | UncheckCommand | SelectCommand | FocusCommand | BlurCommand | HoverCommand | ScrollCommand | ScrollIntoViewCommand | SnapshotCommand | QuerySelectorCommand | QuerySelectorAllCommand | GetTextCommand | GetAttributeCommand | GetPropertyCommand | GetBoundingBoxCommand | IsVisibleCommand | IsEnabledCommand | IsCheckedCommand | PressCommand | KeyDownCommand | KeyUpCommand | WaitCommand | EvaluateCommand | ValidateElementCommand | ValidateRefsCommand | HighlightCommand | ClearHighlightCommand;
+export interface SuccessResponse<T = unknown> {
+    id: string;
+    success: true;
+    data: T;
+}
+/**
+ * Error response with structured data for AI agents
+ *
+ * Includes both human-readable error messages and machine-readable
+ * error codes, context, and recovery suggestions.
+ */
+export interface ErrorResponse {
+    id: string;
+    success: false;
+    /** Human-readable error message */
+    error: string;
+    /** Machine-readable error code (optional) */
+    errorCode?: string;
+    /** Structured error context (optional) */
+    errorContext?: {
+        selector?: string;
+        expectedType?: string;
+        actualType?: string;
+        elementState?: {
+            attached: boolean;
+            visible: boolean;
+            enabled: boolean;
+        };
+        availableActions?: string[];
+        similarSelectors?: Array<{
+            selector: string;
+            role: string;
+            name: string;
+        }>;
+        nearbyElements?: Array<{
+            ref: string;
+            role: string;
+            name: string;
+        }>;
+        [key: string]: any;
+    };
+    /** Actionable recovery suggestions (optional) */
+    suggestions?: string[];
+}
+export type Response<T = unknown> = SuccessResponse<T> | ErrorResponse;
+/**
+ * Response data for validateElement command
+ */
+export interface ValidateElementResponse {
+    /** Whether element is compatible with requested capabilities */
+    compatible: boolean;
+    /** Actual element role/tag */
+    actualRole: string;
+    /** Actual element type (for inputs) */
+    actualType?: string;
+    /** Capabilities this element supports */
+    capabilities: string[];
+    /** Current element state */
+    state: {
+        visible: boolean;
+        enabled: boolean;
+        attached: boolean;
+    };
+    /** Suggestion if not compatible */
+    suggestion?: string;
+}
+/**
+ * Response data for validateRefs command
+ */
+export interface ValidateRefsResponse {
+    /** List of valid refs */
+    valid: string[];
+    /** List of invalid refs */
+    invalid: string[];
+    /** Reasons why each ref is invalid */
+    reasons: Record<string, string>;
+}
+export interface SnapshotNode {
+    role: string;
+    name?: string;
+    ref?: string;
+    value?: string;
+    checked?: boolean;
+    disabled?: boolean;
+    children?: SnapshotNode[];
+}
+export interface SnapshotData {
+    tree: string;
+    refs: Record<string, {
+        selector: string;
+        role: string;
+        name?: string;
+        bbox?: BoundingBox;
+        inViewport?: boolean;
+        importance?: 'primary' | 'secondary' | 'utility';
+        context?: string;
+    }>;
+    metadata?: {
+        totalInteractiveElements?: number;
+        capturedElements?: number;
+        quality?: 'high' | 'medium' | 'low';
+        depthLimited?: boolean;
+        warnings?: string[];
+    };
+}
+export interface BoundingBox {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export interface RefMap {
+    get(ref: string): Element | null;
+    set(ref: string, element: Element): void;
+    clear(): void;
+    generateRef(element: Element): string;
+}
+//# sourceMappingURL=types.d.ts.map

package/packages/core/dist/types.js

@@ -0,0 +1,7 @@
+/**
+ * @btcp/core - Type definitions
+ *
+ * Core types for DOM automation commands and responses.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/packages/extension/dist/background.d.ts

@@ -0,0 +1,227 @@
+/**
+ * @btcp/extension - Background Script
+ *
+ * Contains BrowserAgent - the high-level orchestrator that runs in the
+ * extension's background/service worker context.
+ *
+ * BrowserAgent manages:
+ * - Tab lifecycle (create, close, switch, list)
+ * - Navigation (goto, back, forward, reload)
+ * - Screenshots (chrome.tabs.captureVisibleTab)
+ * - Session state
+ * - Routing DOM commands to ContentAgents in target tabs
+ */
+import type { Command, ExtensionCommand, Response, TabInfo, ChromeTab } from './types.js';
+/**
+ * TabHandle - Interface for interacting with a specific tab
+ *
+ * Returned by BackgroundAgent.tab(tabId) for tab-specific operations.
+ */
+export interface TabHandle {
+    readonly tabId: number;
+    execute(command: Command): Promise<Response>;
+    snapshot(options?: {
+        selector?: string;
+        maxDepth?: number;
+    }): Promise<Response>;
+    click(selector: string): Promise<Response>;
+    fill(selector: string, value: string): Promise<Response>;
+    type(selector: string, text: string, options?: {
+        clear?: boolean;
+    }): Promise<Response>;
+    getText(selector: string): Promise<Response>;
+    isVisible(selector: string): Promise<Response>;
+}
+/**
+ * BackgroundAgent - High-level browser automation orchestrator
+ *
+ * Runs in the extension's background script/service worker.
+ * Manages browser-level operations and routes DOM commands to
+ * ContentAgent instances running in content scripts.
+ *
+ * @example Single tab (default - uses activeTabId)
+ * ```typescript
+ * const agent = new BackgroundAgent();
+ * await agent.navigate('https://example.com');
+ * await agent.execute({ id: '1', action: 'click', selector: '#submit' });
+ * ```
+ *
+ * @example Multi-tab with explicit tabId
+ * ```typescript
+ * const agent = new BackgroundAgent();
+ *
+ * // Open two tabs
+ * const tab1 = await agent.newTab({ url: 'https://google.com' });
+ * const tab2 = await agent.newTab({ url: 'https://github.com', active: false });
+ *
+ * // Interact with specific tabs without switching
+ * await agent.tab(tab1.id).click('#search');
+ * await agent.tab(tab2.id).snapshot();
+ *
+ * // Or specify tabId in command
+ * await agent.execute({ id: '1', action: 'snapshot' }, { tabId: tab2.id });
+ * ```
+ */
+export declare class BackgroundAgent {
+    private activeTabId;
+    private sessionManager;
+    private heartbeatInterval;
+    private readonly HEARTBEAT_INTERVAL;
+    constructor();
+    private initActiveTab;
+    /**
+     * Get the current active tab ID
+     */
+    getActiveTabId(): number | null;
+    /**
+     * Set the active tab ID (for manual control)
+     */
+    setActiveTabId(tabId: number): void;
+    /**
+     * Get a handle for interacting with a specific tab
+     *
+     * This allows you to send commands to any tab without switching the active tab.
+     *
+     * @example
+     * ```typescript
+     * const tab2Handle = browser.tab(tab2.id);
+     * await tab2Handle.snapshot();
+     * await tab2Handle.click('@ref:5');
+     * ```
+     */
+    tab(tabId: number): TabHandle;
+    /**
+     * Get the currently active tab (only if in session)
+     */
+    getActiveTab(): Promise<ChromeTab | null>;
+    /**
+     * List all tabs (only in session group)
+     */
+    listTabs(): Promise<TabInfo[]>;
+    /**
+     * Create a new tab (only in session group)
+     */
+    newTab(options?: {
+        url?: string;
+        active?: boolean;
+    }): Promise<TabInfo>;
+    /**
+     * Check if a tab is in the active session
+     */
+    private isTabInSession;
+    /**
+     * Close a tab (only if in session)
+     */
+    closeTab(tabId?: number): Promise<void>;
+    /**
+     * Switch to a tab (only if in session)
+     */
+    switchTab(tabId: number): Promise<void>;
+    /**
+     * Navigate to a URL (only in session tabs)
+     */
+    navigate(url: string, options?: {
+        waitUntil?: 'load' | 'domcontentloaded';
+    }): Promise<void>;
+    /**
+     * Go back in history
+     */
+    back(): Promise<void>;
+    /**
+     * Go forward in history
+     */
+    forward(): Promise<void>;
+    /**
+     * Reload the current page
+     */
+    reload(options?: {
+        bypassCache?: boolean;
+    }): Promise<void>;
+    /**
+     * Get the current URL
+     */
+    getUrl(): Promise<string>;
+    /**
+     * Get the page title
+     */
+    getTitle(): Promise<string>;
+    /**
+     * Capture a screenshot of the visible tab
+     */
+    screenshot(options?: {
+        format?: 'png' | 'jpeg';
+        quality?: number;
+    }): Promise<string>;
+    /**
+     * Execute a command - routes to appropriate handler
+     *
+     * Browser-level commands (navigate, screenshot, tabs) are handled here.
+     * DOM-level commands are forwarded to the ContentAgent in the target tab.
+     *
+     * @param command - The command to execute
+     * @param options - Optional settings including target tabId
+     *
+     * @example Default (active tab)
+     * ```typescript
+     * await browser.execute({ id: '1', action: 'snapshot' });
+     * ```
+     *
+     * @example Specific tab
+     * ```typescript
+     * await browser.execute({ id: '1', action: 'snapshot' }, { tabId: 123 });
+     * ```
+     */
+    execute(command: Command, options?: {
+        tabId?: number;
+    }): Promise<Response>;
+    /**
+     * Send a command to the ContentAgent in a specific tab
+     */
+    sendToContentAgent(command: Command, tabId?: number): Promise<Response>;
+    /**
+     * Send message with automatic content script re-injection on failure
+     */
+    private sendMessageWithRetry;
+    /**
+     * Re-inject content script into a tab (for recovery from frozen state)
+     */
+    private reinjectContentScript;
+    /**
+     * Start heartbeat to monitor session tabs
+     */
+    private startHeartbeat;
+    /**
+     * Stop heartbeat (for cleanup)
+     * Note: Currently not called as service workers are terminated by Chrome
+     * Could be used if explicit cleanup is needed in the future
+     */
+    private _stopHeartbeat;
+    /**
+     * Ping all session tabs to check health
+     */
+    private pingSessionTabs;
+    private isExtensionCommand;
+    private executeExtensionCommand;
+    private waitForTabLoad;
+}
+/**
+ * Get or create the BackgroundAgent singleton
+ */
+export declare function getBackgroundAgent(): BackgroundAgent;
+/**
+ * @deprecated Use getBackgroundAgent instead
+ */
+export declare const getBrowserAgent: typeof getBackgroundAgent;
+/**
+ * Set up the message listener for the background script
+ * Call this once in your background.ts to enable command routing
+ */
+export declare function setupMessageListener(): void;
+export declare const handleCommand: (command: Command, _tabId?: number) => Promise<Response>;
+export declare const executeExtensionCommand: (command: ExtensionCommand) => Promise<Response>;
+export declare const sendToContentScript: (_tabId: number, command: Command) => Promise<Response>;
+/**
+ * @deprecated Use BackgroundAgent instead
+ */
+export declare const BrowserAgent: typeof BackgroundAgent;
+//# sourceMappingURL=background.d.ts.map