autokap 1.0.6 → 1.0.7

package/dist/remote-browser.d.ts

@@ -0,0 +1,191 @@
+/**
+ * RemoteBrowser — Server-side proxy that mirrors the Browser interface
+ * but forwards every method call to a remote CLI via WebSocket.
+ *
+ * Used by the WS server to control a Playwright browser running on the user's machine.
+ */
+import type { WebSocket } from 'ws';
+import type { PageState, PageStateLite, InteractiveElement, BrowserStorageState, BrowserSessionStorageState, VideoPageSignals, OutscaleConfig, SelectorValidationResult } from './types.js';
+import type { BrowserObservation, BrowserReaction, BrowserSelectorProbe, BrowserStorageHintWriteParams, BrowserVerificationBundle, BrowserVideoVerificationBundle } from './browser.js';
+export interface WsCommand {
+    id: string;
+    method: string;
+    params: Record<string, unknown>;
+}
+export interface WsResult {
+    id: string;
+    result?: unknown;
+    error?: string;
+}
+export interface WsEvent {
+    type: 'done' | 'error' | 'progress';
+    message?: string;
+    summary?: {
+        successes: number;
+        total: number;
+    };
+}
+export declare const REMOTE_BROWSER_PROTOCOL_VERSION = 2;
+export interface RemoteBrowserClientInfo {
+    version?: string;
+    protocolVersion?: number;
+}
+export interface WsClientHello {
+    type: 'client_hello';
+    client: RemoteBrowserClientInfo;
+}
+export declare function buildRemoteBrowserClientHello(version: string): WsClientHello;
+export declare function getRemoteBrowserCompatibilityError(client: RemoteBrowserClientInfo | null | undefined): string | null;
+/**
+ * Lightweight proxy for `browser.currentPage` — only exposes the methods
+ * that `runAgent` actually calls: `.url()`, `.viewportSize()`, `.title()`.
+ */
+declare class RemotePageProxy {
+    private callFn;
+    constructor(callFn: (method: string, params: Record<string, unknown>) => Promise<unknown>);
+    url(): string;
+    viewportSize(): {
+        width: number;
+        height: number;
+    } | null;
+    title(): Promise<string>;
+    _cachedUrl: string;
+    _cachedViewport: {
+        width: number;
+        height: number;
+    } | null;
+    _sync(): Promise<void>;
+}
+export declare class RemoteBrowser {
+    private ws;
+    private pending;
+    private idCounter;
+    private _pageProxy;
+    /** Proxy for browser.currentPage — provides url(), viewportSize(), title() */
+    get currentPage(): RemotePageProxy;
+    constructor(ws: WebSocket);
+    launch(): Promise<void>;
+    close(): Promise<void>;
+    closeContext(): Promise<void>;
+    recreateContext(options: {
+        viewport?: {
+            width: number;
+            height: number;
+        };
+        deviceScaleFactor?: number;
+        lang?: string;
+        colorScheme?: 'light' | 'dark';
+        storageState?: BrowserStorageState;
+    }): Promise<void>;
+    setDeviceScaleFactor(deviceScaleFactor: number): Promise<void>;
+    navigateTo(url: string): Promise<void>;
+    addCookies(cookies: Array<Record<string, unknown>>): Promise<void>;
+    takeScreenshot(): Promise<Buffer>;
+    takeScreenshotForAI(options?: {
+        timeoutMs?: number;
+    }): Promise<Buffer>;
+    screenshotElement(index: number, padding?: number): Promise<Buffer>;
+    screenshotRegion(x: number, y: number, width: number, height: number, padding?: number): Promise<Buffer>;
+    screenshotBySelector(selector: string, outscale?: OutscaleConfig | number): Promise<{
+        buffer: Buffer;
+        validation: SelectorValidationResult;
+    }>;
+    screenshotByRegion(region: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    }, outscale?: OutscaleConfig | number): Promise<Buffer>;
+    getPageState(opts?: {
+        skipAnnotation?: boolean;
+    }): Promise<PageState>;
+    getPageStateLite(): Promise<PageStateLite>;
+    getAccessibilityTree(options?: {
+        timeoutMs?: number;
+    }): Promise<string>;
+    getInteractiveElements(options?: {
+        timeoutMs?: number;
+    }): Promise<InteractiveElement[]>;
+    getSimplifiedDOM(): Promise<string>;
+    captureObservation(): Promise<BrowserObservation>;
+    capturePageSignals(options?: {
+        timeoutMs?: number;
+    }): Promise<VideoPageSignals>;
+    captureVerificationBundle(options?: {
+        settleMs?: number;
+        maxAttempts?: number;
+    }): Promise<BrowserVerificationBundle>;
+    captureVideoVerificationBundle(options?: {
+        settleMs?: number;
+        maxAttempts?: number;
+        helperTimeoutMs?: number;
+    }): Promise<BrowserVideoVerificationBundle>;
+    clickByIndex(index: number): Promise<void>;
+    clickBySelector(selector: string, options?: {
+        force?: boolean;
+    }): Promise<void>;
+    clickByCoordinates(x: number, y: number): Promise<void>;
+    hoverByIndex(index: number): Promise<void>;
+    hoverBySelector(selector: string): Promise<void>;
+    hoverByCoordinates(x: number, y: number): Promise<void>;
+    typeText(text: string, options?: {
+        index?: number;
+        selector?: string;
+        clearFirst?: boolean;
+    }): Promise<void>;
+    selectOption(options: {
+        index?: number;
+        selector?: string;
+        optionLabel?: string;
+        optionValue?: string;
+        optionIndex?: number;
+    }): Promise<void>;
+    scroll(direction: string, amount?: number, selector?: string): Promise<void>;
+    scrollElementIntoView(index: number, options?: {
+        align?: 'start' | 'center' | 'end';
+        margin?: number;
+    }): Promise<void>;
+    safeExpand(options: {
+        index?: number;
+        selector?: string;
+        expectedText?: string;
+        expectedSelector?: string;
+    }): Promise<void>;
+    pressKey(key: string): Promise<void>;
+    searchText(query: string): Promise<Array<Record<string, unknown>>>;
+    waitForPageReaction(before?: BrowserObservation, options?: {
+        timeoutMs?: number;
+        settleMs?: number;
+        idleGraceMs?: number;
+    }): Promise<BrowserReaction>;
+    wait(ms: number): Promise<void>;
+    dismissOverlays(): Promise<{
+        dismissed: boolean;
+        method: string | null;
+    }>;
+    setColorScheme(scheme: 'light' | 'dark'): Promise<void>;
+    setLanguage(lang: string): Promise<void>;
+    resizeViewport(width: number, height: number): Promise<void>;
+    forceLoadLazyImages(options?: {
+        timeout?: number;
+    }): Promise<void>;
+    exportStorageState(): Promise<BrowserStorageState>;
+    exportSessionStorage(): Promise<BrowserSessionStorageState>;
+    reloadCurrentPage(options?: {
+        waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit';
+        timeout?: number;
+    }): Promise<void>;
+    writeStorageHintCandidate(params: BrowserStorageHintWriteParams): Promise<boolean>;
+    probeSelector(selector: string): Promise<BrowserSelectorProbe | null>;
+    prepareSessionStorage(bundle: BrowserSessionStorageState | undefined, options?: {
+        replace?: boolean;
+    }): Promise<void>;
+    observeNetworkRequests(url: string, waitMs?: number): Promise<Array<Record<string, unknown>>>;
+    setupRouteInterception(mocks: Array<Record<string, unknown>>): Promise<void>;
+    clearRouteInterception(): Promise<void>;
+    private call;
+    private callAndSyncPageInfo;
+    sendEvent(event: WsEvent): void;
+    destroy(): void;
+}
+export {};

package/dist/remote-browser.js

@@ -0,0 +1,305 @@
+/**
+ * RemoteBrowser — Server-side proxy that mirrors the Browser interface
+ * but forwards every method call to a remote CLI via WebSocket.
+ *
+ * Used by the WS server to control a Playwright browser running on the user's machine.
+ */
+export const REMOTE_BROWSER_PROTOCOL_VERSION = 2;
+export function buildRemoteBrowserClientHello(version) {
+    return {
+        type: 'client_hello',
+        client: {
+            version,
+            protocolVersion: REMOTE_BROWSER_PROTOCOL_VERSION,
+        },
+    };
+}
+export function getRemoteBrowserCompatibilityError(client) {
+    if (!client) {
+        return 'Installed AutoKap CLI is too old for this server. Update the CLI and retry.';
+    }
+    if ((client.protocolVersion ?? 0) < REMOTE_BROWSER_PROTOCOL_VERSION) {
+        const versionLabel = client.version ? ` ${client.version}` : '';
+        return `Installed AutoKap CLI${versionLabel} is too old for this server. Update the CLI and retry.`;
+    }
+    return null;
+}
+const DEFAULT_TIMEOUT_MS = 60_000;
+/**
+ * Lightweight proxy for `browser.currentPage` — only exposes the methods
+ * that `runAgent` actually calls: `.url()`, `.viewportSize()`, `.title()`.
+ */
+class RemotePageProxy {
+    callFn;
+    constructor(callFn) {
+        this.callFn = callFn;
+    }
+    url() {
+        // Synchronous in Playwright but we cache the last known value.
+        // The agent calls this frequently — we use a sync fallback via _cachedUrl.
+        return this._cachedUrl;
+    }
+    viewportSize() {
+        return this._cachedViewport;
+    }
+    async title() {
+        return await this.callFn('getPageTitle', {});
+    }
+    _cachedUrl = '';
+    _cachedViewport = null;
+    async _sync() {
+        const state = await this.callFn('getPageInfo', {});
+        this._cachedUrl = state.url;
+        this._cachedViewport = state.viewport;
+    }
+}
+export class RemoteBrowser {
+    ws;
+    pending = new Map();
+    idCounter = 0;
+    _pageProxy;
+    /** Proxy for browser.currentPage — provides url(), viewportSize(), title() */
+    get currentPage() {
+        return this._pageProxy;
+    }
+    constructor(ws) {
+        this.ws = ws;
+        this._pageProxy = new RemotePageProxy((method, params) => this.call(method, params));
+        this.ws.on('message', (data) => {
+            try {
+                const msg = JSON.parse(typeof data === 'string' ? data : data.toString('utf-8'));
+                const call = this.pending.get(msg.id);
+                if (!call)
+                    return;
+                this.pending.delete(msg.id);
+                clearTimeout(call.timer);
+                if (msg.error) {
+                    call.reject(new Error(msg.error));
+                }
+                else {
+                    call.resolve(msg.result);
+                }
+            }
+            catch { /* ignore malformed messages */ }
+        });
+    }
+    // ── Lifecycle ────────────────────────────────────────────────────
+    async launch() {
+        await this.call('launch', {});
+    }
+    async close() {
+        await this.call('close', {});
+    }
+    async closeContext() {
+        await this.call('closeContext', {});
+    }
+    async recreateContext(options) {
+        await this.call('recreateContext', options);
+        await this._pageProxy._sync();
+    }
+    async setDeviceScaleFactor(deviceScaleFactor) {
+        await this.call('setDeviceScaleFactor', { deviceScaleFactor });
+        await this._pageProxy._sync();
+    }
+    // ── Navigation ───────────────────────────────────────────────────
+    async navigateTo(url) {
+        await this.call('navigateTo', { url });
+        await this._pageProxy._sync();
+    }
+    async addCookies(cookies) {
+        await this.call('addCookies', { cookies });
+    }
+    // ── Screenshots ──────────────────────────────────────────────────
+    async takeScreenshot() {
+        const r = await this.call('takeScreenshot', {});
+        return Buffer.from(r.screenshot, 'base64');
+    }
+    async takeScreenshotForAI(options) {
+        const r = await this.call('takeScreenshotForAI', options ?? {});
+        return Buffer.from(r.screenshot, 'base64');
+    }
+    async screenshotElement(index, padding) {
+        const r = await this.call('screenshotElement', { index, padding });
+        return Buffer.from(r.screenshot, 'base64');
+    }
+    async screenshotRegion(x, y, width, height, padding) {
+        const r = await this.call('screenshotRegion', { x, y, width, height, padding });
+        return Buffer.from(r.screenshot, 'base64');
+    }
+    async screenshotBySelector(selector, outscale) {
+        const r = await this.call('screenshotBySelector', { selector, outscale });
+        return { buffer: Buffer.from(r.buffer, 'base64'), validation: r.validation };
+    }
+    async screenshotByRegion(region, outscale) {
+        const r = await this.call('screenshotByRegion', { region, outscale });
+        return Buffer.from(r.screenshot, 'base64');
+    }
+    // ── Page State ───────────────────────────────────────────────────
+    async getPageState(opts) {
+        const r = await this.call('getPageState', opts ?? {});
+        // Sync page info from the response metadata if available, or fetch separately
+        await this._pageProxy._sync();
+        return {
+            ...r,
+            cleanScreenshot: Buffer.from(r.cleanScreenshot, 'base64'),
+            screenshot: Buffer.from(r.screenshot, 'base64'),
+        };
+    }
+    async getPageStateLite() {
+        return await this.call('getPageStateLite', {});
+    }
+    async getAccessibilityTree(options) {
+        return await this.call('getAccessibilityTree', options ?? {});
+    }
+    async getInteractiveElements(options) {
+        return await this.call('getInteractiveElements', options ?? {});
+    }
+    async getSimplifiedDOM() {
+        return await this.call('getSimplifiedDOM', {});
+    }
+    async captureObservation() {
+        return await this.call('captureObservation', {});
+    }
+    async capturePageSignals(options) {
+        return await this.call('capturePageSignals', options ?? {});
+    }
+    async captureVerificationBundle(options) {
+        const r = await this.call('captureVerificationBundle', options ?? {});
+        return {
+            ...r,
+            screenshot: Buffer.from(r.screenshot, 'base64'),
+        };
+    }
+    async captureVideoVerificationBundle(options) {
+        const r = await this.call('captureVideoVerificationBundle', options ?? {});
+        return {
+            ...r,
+            screenshot: Buffer.from(r.screenshot, 'base64'),
+        };
+    }
+    // ── Interactions ─────────────────────────────────────────────────
+    async clickByIndex(index) {
+        await this.callAndSyncPageInfo('clickByIndex', { index });
+    }
+    async clickBySelector(selector, options) {
+        await this.callAndSyncPageInfo('clickBySelector', { selector, ...options });
+    }
+    async clickByCoordinates(x, y) {
+        await this.callAndSyncPageInfo('clickByCoordinates', { x, y });
+    }
+    async hoverByIndex(index) {
+        await this.call('hoverByIndex', { index });
+    }
+    async hoverBySelector(selector) {
+        await this.call('hoverBySelector', { selector });
+    }
+    async hoverByCoordinates(x, y) {
+        await this.call('hoverByCoordinates', { x, y });
+    }
+    async typeText(text, options) {
+        await this.callAndSyncPageInfo('typeText', { text, ...options });
+    }
+    async selectOption(options) {
+        await this.callAndSyncPageInfo('selectOption', options);
+    }
+    async scroll(direction, amount, selector) {
+        await this.callAndSyncPageInfo('scroll', { direction, amount, selector });
+    }
+    async scrollElementIntoView(index, options) {
+        await this.callAndSyncPageInfo('scrollElementIntoView', { index, ...options });
+    }
+    async safeExpand(options) {
+        await this.callAndSyncPageInfo('safeExpand', options);
+    }
+    async pressKey(key) {
+        await this.callAndSyncPageInfo('pressKey', { key });
+    }
+    async searchText(query) {
+        return await this.call('searchText', { query });
+    }
+    // ── Reactions ────────────────────────────────────────────────────
+    async waitForPageReaction(before, options) {
+        const reaction = await this.call('waitForPageReaction', { before, ...options });
+        await this._pageProxy._sync();
+        return reaction;
+    }
+    // ── UI Manipulation ──────────────────────────────────────────────
+    async wait(ms) {
+        await this.call('wait', { ms });
+    }
+    async dismissOverlays() {
+        const result = await this.call('dismissOverlays', {});
+        await this._pageProxy._sync();
+        return result;
+    }
+    async setColorScheme(scheme) {
+        await this.callAndSyncPageInfo('setColorScheme', { scheme });
+    }
+    async setLanguage(lang) {
+        await this.callAndSyncPageInfo('setLanguage', { lang });
+    }
+    async resizeViewport(width, height) {
+        await this.call('resizeViewport', { width, height });
+        this._pageProxy._cachedViewport = { width, height };
+    }
+    async forceLoadLazyImages(options) {
+        await this.call('forceLoadLazyImages', options ?? {});
+    }
+    // ── Storage ──────────────────────────────────────────────────────
+    async exportStorageState() {
+        return await this.call('exportStorageState', {});
+    }
+    async exportSessionStorage() {
+        return await this.call('exportSessionStorage', {});
+    }
+    async reloadCurrentPage(options) {
+        await this.callAndSyncPageInfo('reloadCurrentPage', options ?? {});
+    }
+    async writeStorageHintCandidate(params) {
+        return await this.call('writeStorageHintCandidate', params);
+    }
+    async probeSelector(selector) {
+        return await this.call('probeSelector', { selector });
+    }
+    async prepareSessionStorage(bundle, options) {
+        await this.call('prepareSessionStorage', { bundle, ...options });
+    }
+    // ── Network ──────────────────────────────────────────────────────
+    async observeNetworkRequests(url, waitMs) {
+        return await this.call('observeNetworkRequests', { url, waitMs });
+    }
+    async setupRouteInterception(mocks) {
+        await this.call('setupRouteInterception', { mocks });
+    }
+    async clearRouteInterception() {
+        await this.call('clearRouteInterception', {});
+    }
+    // ── RPC core ─────────────────────────────────────────────────────
+    call(method, params, timeoutMs = DEFAULT_TIMEOUT_MS) {
+        return new Promise((resolve, reject) => {
+            const id = `cmd_${++this.idCounter}`;
+            const timer = setTimeout(() => {
+                this.pending.delete(id);
+                reject(new Error(`RemoteBrowser: timeout waiting for ${method} (${timeoutMs}ms)`));
+            }, timeoutMs);
+            this.pending.set(id, { resolve, reject, timer });
+            this.ws.send(JSON.stringify({ id, method, params }));
+        });
+    }
+    async callAndSyncPageInfo(method, params) {
+        const result = await this.call(method, params);
+        await this._pageProxy._sync();
+        return result;
+    }
+    sendEvent(event) {
+        this.ws.send(JSON.stringify(event));
+    }
+    destroy() {
+        for (const [, call] of this.pending) {
+            clearTimeout(call.timer);
+            call.reject(new Error('RemoteBrowser: connection closed'));
+        }
+        this.pending.clear();
+    }
+}
+//# sourceMappingURL=remote-browser.js.map

package/dist/security.d.ts

@@ -0,0 +1,20 @@
+import type { ActionType, AgentConfig, CaptureObjective, CaptureRepairCause, InteractiveElement } from './types.js';
+export interface SecurityContext {
+    rootUrl?: string;
+    currentUrl?: string;
+    credentials?: AgentConfig['credentials'];
+    interactiveElements: InteractiveElement[];
+    currentLang?: string;
+    currentTheme?: 'light' | 'dark';
+    runMode?: AgentConfig['runMode'];
+    currentObjective?: CaptureObjective;
+    activeRepairCause?: CaptureRepairCause | null;
+}
+export interface SecurityDecision {
+    allowed: boolean;
+    reason?: string;
+    target?: InteractiveElement | null;
+}
+export declare function evaluateResolvedActionSecurity(action: ActionType, args: Record<string, unknown>, context: SecurityContext, target: InteractiveElement | null): SecurityDecision;
+export declare function evaluateActionSecurity(action: ActionType, args: Record<string, unknown>, context: SecurityContext): SecurityDecision;
+export declare function describeSecurityTarget(target: InteractiveElement | null | undefined): string;