@agentrhq/webcmd 0.1.1 → 0.2.0

package/dist/src/browser/ax-snapshot.d.ts

@@ -0,0 +1,37 @@
+/**
+ * AX-backed browser snapshot prototype.
+ *
+ * This is intentionally additive to the current DOM snapshot. It learns from
+ * agent-browser's accessibility-tree refs without changing default `state`
+ * output until the AX path proves itself on fixtures and real SaaS workflows.
+ */
+export interface BrowserRef {
+    ref: string;
+    backendNodeId?: number;
+    role: string;
+    name: string;
+    nth?: number;
+    frame?: {
+        frameId?: string;
+        sessionId?: string;
+        url?: string;
+        targetUrl?: string;
+    };
+}
+export interface AxSnapshotTree {
+    tree: unknown;
+    frame?: BrowserRef['frame'];
+}
+export interface AxSnapshotBuildResult {
+    text: string;
+    refs: Map<string, BrowserRef>;
+}
+export declare function buildAxSnapshot(axTree: unknown, opts?: {
+    maxDepth?: number;
+    interactiveOnly?: boolean;
+}): AxSnapshotBuildResult;
+export declare function buildAxSnapshotFromTrees(trees: AxSnapshotTree[], opts?: {
+    maxDepth?: number;
+    interactiveOnly?: boolean;
+}): AxSnapshotBuildResult;
+export declare function findAxRefReplacement(axTree: unknown, ref: BrowserRef): BrowserRef | null;

package/dist/src/browser/ax-snapshot.js

@@ -0,0 +1,217 @@
+/**
+ * AX-backed browser snapshot prototype.
+ *
+ * This is intentionally additive to the current DOM snapshot. It learns from
+ * agent-browser's accessibility-tree refs without changing default `state`
+ * output until the AX path proves itself on fixtures and real SaaS workflows.
+ */
+const INTERACTIVE_ROLES = new Set([
+    'button',
+    'link',
+    'textbox',
+    'searchbox',
+    'checkbox',
+    'radio',
+    'combobox',
+    'listbox',
+    'menuitem',
+    'menuitemcheckbox',
+    'menuitemradio',
+    'option',
+    'slider',
+    'spinbutton',
+    'switch',
+    'tab',
+    'treeitem',
+]);
+const CONTENT_ROLES = new Set([
+    'article',
+    'cell',
+    'columnheader',
+    'gridcell',
+    'heading',
+    'listitem',
+    'main',
+    'navigation',
+    'region',
+    'rowheader',
+]);
+const STRUCTURAL_ROLES = new Set([
+    'generic',
+    'group',
+    'list',
+    'none',
+    'presentation',
+    'RootWebArea',
+    'WebArea',
+]);
+export function buildAxSnapshot(axTree, opts = {}) {
+    return buildAxSnapshotFromTrees([{ tree: axTree }], opts);
+}
+export function buildAxSnapshotFromTrees(trees, opts = {}) {
+    const lines = ['source: ax', '---'];
+    const refs = new Map();
+    let nextRef = 1;
+    for (const [index, entry] of trees.entries()) {
+        if (index > 0) {
+            const label = entry.frame?.url ? JSON.stringify(entry.frame.url) : JSON.stringify(entry.frame?.frameId ?? `frame:${index}`);
+            lines.push(`frame ${label}:`);
+        }
+        nextRef = renderAxTree(entry.tree, lines, refs, nextRef, {
+            ...opts,
+            frame: entry.frame,
+            baseDepth: index > 0 ? 1 : 0,
+        });
+    }
+    lines.push('---');
+    lines.push(`interactive: ${refs.size}`);
+    return { text: lines.join('\n'), refs };
+}
+function renderAxTree(axTree, lines, refs, nextRef, opts) {
+    const rawNodes = Array.isArray(axTree?.nodes)
+        ? axTree.nodes
+        : [];
+    const nodes = rawNodes.filter((node) => node && !node.ignored);
+    const byId = new Map();
+    const parentIds = new Set();
+    for (const node of nodes) {
+        if (typeof node.nodeId === 'string')
+            byId.set(node.nodeId, node);
+        for (const childId of node.childIds ?? [])
+            parentIds.add(childId);
+    }
+    const roots = nodes.filter((node) => {
+        if (!node.nodeId)
+            return false;
+        const role = axString(node.role);
+        return !parentIds.has(node.nodeId) || role === 'RootWebArea' || role === 'WebArea';
+    });
+    const root = roots[0] ?? nodes[0];
+    const maxDepth = Math.max(1, Math.min(Number(opts.maxDepth) || 50, 200));
+    const roleNameCounts = countRoleNames(nodes);
+    const roleNameSeen = new Map();
+    function render(node, depth) {
+        if (!node || depth > maxDepth)
+            return false;
+        const role = axString(node.role) || 'generic';
+        const name = cleanText(axString(node.name));
+        const value = cleanText(axString(node.value) || propertyValue(node, 'value'));
+        const disabled = propertyValue(node, 'disabled');
+        const checked = propertyValue(node, 'checked');
+        const expanded = propertyValue(node, 'expanded');
+        const selected = propertyValue(node, 'selected');
+        const refEligible = shouldRef(role, name, node.backendDOMNodeId);
+        const shouldShowSelf = refEligible
+            || !!name
+            || !!value
+            || CONTENT_ROLES.has(role)
+            || (!opts.interactiveOnly && !STRUCTURAL_ROLES.has(role));
+        const childStart = lines.length;
+        let hasVisibleChild = false;
+        for (const childId of node.childIds ?? []) {
+            if (render(byId.get(childId), depth + 1))
+                hasVisibleChild = true;
+        }
+        if (!shouldShowSelf && !hasVisibleChild) {
+            lines.length = childStart;
+            return false;
+        }
+        if (shouldShowSelf) {
+            const indent = '  '.repeat(depth);
+            const parts = [];
+            let prefix = '';
+            if (refEligible) {
+                const ref = String(nextRef++);
+                prefix = `[${ref}]`;
+                const key = roleNameKey(role, name);
+                const seen = roleNameSeen.get(key) ?? 0;
+                roleNameSeen.set(key, seen + 1);
+                refs.set(ref, {
+                    ref,
+                    backendNodeId: node.backendDOMNodeId,
+                    role,
+                    name,
+                    ...(roleNameCounts.get(key) > 1 ? { nth: seen } : {}),
+                    ...(opts.frame ? { frame: opts.frame } : {}),
+                });
+            }
+            if (name)
+                parts.push(JSON.stringify(name));
+            if (value && value !== name)
+                parts.push(`value=${JSON.stringify(value)}`);
+            if (checked)
+                parts.push(`checked=${checked}`);
+            if (expanded)
+                parts.push(`expanded=${expanded}`);
+            if (selected)
+                parts.push(`selected=${selected}`);
+            if (disabled === 'true')
+                parts.push('disabled');
+            lines.splice(childStart, 0, `${indent}${prefix}${role}${parts.length ? ` ${parts.join(' ')}` : ''}`);
+        }
+        return true;
+    }
+    render(root, opts.baseDepth);
+    return nextRef;
+}
+export function findAxRefReplacement(axTree, ref) {
+    const nodes = Array.isArray(axTree?.nodes)
+        ? axTree.nodes
+        : [];
+    const targetNth = ref.nth ?? 0;
+    let seen = 0;
+    for (const node of nodes) {
+        if (!node || node.ignored)
+            continue;
+        const role = axString(node.role);
+        const name = cleanText(axString(node.name));
+        if (role !== ref.role || name !== ref.name)
+            continue;
+        if (seen === targetNth) {
+            if (typeof node.backendDOMNodeId !== 'number')
+                return null;
+            return { ...ref, backendNodeId: node.backendDOMNodeId };
+        }
+        seen++;
+    }
+    return null;
+}
+function shouldRef(role, name, backendNodeId) {
+    if (typeof backendNodeId !== 'number')
+        return false;
+    if (INTERACTIVE_ROLES.has(role))
+        return true;
+    return CONTENT_ROLES.has(role) && !!name;
+}
+function countRoleNames(nodes) {
+    const counts = new Map();
+    for (const node of nodes) {
+        if (!node || node.ignored)
+            continue;
+        const role = axString(node.role);
+        const name = cleanText(axString(node.name));
+        if (!shouldRef(role, name, node.backendDOMNodeId))
+            continue;
+        const key = roleNameKey(role, name);
+        counts.set(key, (counts.get(key) ?? 0) + 1);
+    }
+    return counts;
+}
+function roleNameKey(role, name) {
+    return `${role}\u0000${name}`;
+}
+function axString(value) {
+    const raw = value?.value;
+    if (typeof raw === 'string')
+        return raw;
+    if (typeof raw === 'number' || typeof raw === 'boolean')
+        return String(raw);
+    return '';
+}
+function propertyValue(node, name) {
+    const prop = node.properties?.find((candidate) => candidate.name === name);
+    return axString(prop?.value);
+}
+function cleanText(value) {
+    return value.replace(/\s+/g, ' ').trim().slice(0, 160);
+}

package/dist/src/browser/ax-snapshot.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/browser/ax-snapshot.test.js

@@ -0,0 +1,91 @@
+import { describe, expect, it } from 'vitest';
+import { buildAxSnapshot, buildAxSnapshotFromTrees, findAxRefReplacement } from './ax-snapshot.js';
+describe('AX snapshot prototype', () => {
+    it('builds compact refs from Accessibility.getFullAXTree output', () => {
+        const result = buildAxSnapshot({
+            nodes: [
+                { nodeId: '1', role: { value: 'RootWebArea' }, name: { value: 'Example' }, childIds: ['2', '3'] },
+                { nodeId: '2', role: { value: 'heading' }, name: { value: 'Expenses' }, backendDOMNodeId: 20 },
+                {
+                    nodeId: '3',
+                    role: { value: 'combobox' },
+                    name: { value: 'Category' },
+                    backendDOMNodeId: 30,
+                    properties: [{ name: 'expanded', value: { value: false } }],
+                },
+            ],
+        });
+        expect(result.text).toContain('source: ax');
+        expect(result.text).toContain('[1]heading "Expenses"');
+        expect(result.text).toContain('[2]combobox "Category" expanded=false');
+        expect(result.refs.get('2')).toEqual({
+            ref: '2',
+            backendNodeId: 30,
+            role: 'combobox',
+            name: 'Category',
+        });
+    });
+    it('tracks nth only for duplicate role/name pairs', () => {
+        const result = buildAxSnapshot({
+            nodes: [
+                { nodeId: '1', role: { value: 'RootWebArea' }, childIds: ['2', '3'] },
+                { nodeId: '2', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 10 },
+                { nodeId: '3', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 11 },
+            ],
+        });
+        expect(result.refs.get('1')).toMatchObject({ role: 'button', name: 'Save', nth: 0 });
+        expect(result.refs.get('2')).toMatchObject({ role: 'button', name: 'Save', nth: 1 });
+    });
+    it('finds stale ref replacements by role/name/nth', () => {
+        const replacement = findAxRefReplacement({
+            nodes: [
+                { nodeId: '1', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 10 },
+                { nodeId: '2', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 42 },
+            ],
+        }, {
+            ref: '2',
+            role: 'button',
+            name: 'Save',
+            backendNodeId: 11,
+            nth: 1,
+        });
+        expect(replacement).toMatchObject({ ref: '2', backendNodeId: 42, role: 'button', name: 'Save', nth: 1 });
+    });
+    it('combines frame AX trees while keeping ref metadata frame-scoped', () => {
+        const result = buildAxSnapshotFromTrees([
+            {
+                tree: {
+                    nodes: [
+                        { nodeId: '1', role: { value: 'RootWebArea' }, childIds: ['2'] },
+                        { nodeId: '2', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 10 },
+                    ],
+                },
+            },
+            {
+                frame: { frameId: 'frame-1', url: 'https://app.example/embed' },
+                tree: {
+                    nodes: [
+                        { nodeId: '1', role: { value: 'RootWebArea' }, childIds: ['2'] },
+                        { nodeId: '2', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 20 },
+                    ],
+                },
+            },
+        ]);
+        expect(result.text).toContain('[1]button "Save"');
+        expect(result.text).toContain('frame "https://app.example/embed":');
+        expect(result.text).toContain('  [2]button "Save"');
+        expect(result.refs.get('1')).toEqual({
+            ref: '1',
+            backendNodeId: 10,
+            role: 'button',
+            name: 'Save',
+        });
+        expect(result.refs.get('2')).toEqual({
+            ref: '2',
+            backendNodeId: 20,
+            role: 'button',
+            name: 'Save',
+            frame: { frameId: 'frame-1', url: 'https://app.example/embed' },
+        });
+    });
+});

package/dist/src/browser/base-page.d.ts

@@ -0,0 +1,151 @@
+/**
+ * BasePage — shared IPage method implementations for DOM helpers.
+ *
+ * Both Page (daemon-backed) and CDPPage (direct CDP) execute JS the same way
+ * for DOM operations. This base class deduplicates ~200 lines of identical
+ * click/type/scroll/wait/snapshot/interceptor methods.
+ *
+ * Subclasses implement the transport-specific methods: goto, evaluate,
+ * getCookies, screenshot, tabs, etc.
+ */
+import type { BrowserCookie, BrowserEvaluateFunction, FetchJsonOptions, IPage, ScreenshotOptions, SnapshotOptions, WaitOptions } from '../types.js';
+import { type ResolveOptions, type TargetMatchLevel } from './target-resolver.js';
+export interface ResolveSuccess {
+    matches_n: number;
+    /**
+     * Cascading stale-ref tier the resolver traversed. Callers surface this to
+     * agents so `stable` / `reidentified` hits are visibly distinct from a
+     * clean `exact` match — the page changed, the action still succeeded.
+     */
+    match_level: TargetMatchLevel;
+}
+export interface FillTextResult extends ResolveSuccess {
+    filled: boolean;
+    verified: boolean;
+    expected: string;
+    actual: string;
+    length: number;
+    mode?: 'input' | 'textarea' | 'contenteditable';
+}
+export interface SetCheckedResult extends ResolveSuccess {
+    checked: boolean;
+    changed: boolean;
+    kind?: string;
+}
+export interface UploadFilesResult extends ResolveSuccess {
+    uploaded: boolean;
+    files: number;
+    file_names: string[];
+    target: string;
+    multiple?: boolean;
+    accept?: string;
+}
+export interface DragResult {
+    dragged: boolean;
+    source: string;
+    target: string;
+    source_matches_n: number;
+    target_matches_n: number;
+    source_match_level: TargetMatchLevel;
+    target_match_level: TargetMatchLevel;
+}
+export declare abstract class BasePage implements IPage {
+    protected _lastUrl: string | null;
+    /** Cached previous snapshot hashes for incremental diff marking */
+    private _prevSnapshotHashes;
+    private _cdpTargetMarkerSeq;
+    private _axRefs;
+    abstract goto(url: string, options?: {
+        waitUntil?: 'load' | 'none';
+        settleMs?: number;
+        allowBoundNavigation?: boolean;
+    }): Promise<void>;
+    abstract evaluate<T = unknown>(js: string): Promise<T>;
+    abstract evaluate<Args extends unknown[], T>(fn: BrowserEvaluateFunction<Args, T>, ...args: Args): Promise<Awaited<T>>;
+    /**
+     * Safely evaluate JS with pre-serialized arguments.
+     * Each key in `args` becomes a `const` declaration with JSON-serialized value,
+     * wrapped in a lexical block to avoid polluting the global execution context.
+     *
+     * Why a block: Chrome's Runtime.evaluate shares a single global context per page.
+     * Top-level `const` declarations persist across calls, so re-declaring the same
+     * variable name (e.g. `markerAttr` in both click resolution and file upload)
+     * throws "SyntaxError: Identifier has already been declared". A block keeps
+     * the args scoped without forcing callers to pass expression-only code.
+     *
+     * Usage:
+     *   page.evaluateWithArgs(`(async () => { return sym; })()`, { sym: userInput })
+     */
+    evaluateWithArgs(js: string, args: Record<string, unknown>): Promise<unknown>;
+    fetchJson(url: string, opts?: FetchJsonOptions): Promise<unknown>;
+    abstract getCookies(opts?: {
+        domain?: string;
+        url?: string;
+    }): Promise<BrowserCookie[]>;
+    abstract screenshot(options?: ScreenshotOptions): Promise<string>;
+    annotatedScreenshot(options?: ScreenshotOptions): Promise<string>;
+    abstract tabs(): Promise<unknown[]>;
+    abstract selectTab(target: number | string): Promise<void>;
+    click(ref: string, opts?: ResolveOptions): Promise<ResolveSuccess>;
+    /** Uses native CDP click support when the concrete page exposes it. */
+    protected tryNativeClick(x: number, y: number): Promise<boolean>;
+    protected tryNativeMouseMove(x: number, y: number): Promise<boolean>;
+    protected tryNativeDoubleClick(x: number, y: number): Promise<boolean>;
+    protected tryNativeDrag(from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }): Promise<boolean>;
+    protected tryClickAxRef(ref: string): Promise<ResolveSuccess | null>;
+    private resolveAxRefPoint;
+    private axBoxCenter;
+    /** Uses native CDP text insertion when the concrete page exposes it. */
+    protected tryNativeType(text: string): Promise<boolean>;
+    /** Uses native CDP key events when the concrete page exposes them. */
+    protected tryNativeKeyPress(key: string, modifiers: string[]): Promise<boolean>;
+    protected isResolvedFocused(): Promise<boolean>;
+    /**
+     * Run a DOM-domain CDP command against `window.__resolved`.
+     *
+     * CDP DOM.focus / DOM.scrollIntoViewIfNeeded need a nodeId, while our
+     * resolver stores the live Element in page JS. Bridge the two worlds with a
+     * short-lived marker attribute, then query it through CDP.
+     */
+    protected tryCdpOnResolvedElement(method: 'DOM.focus' | 'DOM.scrollIntoViewIfNeeded'): Promise<boolean>;
+    typeText(ref: string, text: string, opts?: ResolveOptions): Promise<ResolveSuccess>;
+    hover(ref: string, opts?: ResolveOptions): Promise<ResolveSuccess>;
+    focus(ref: string, opts?: ResolveOptions): Promise<ResolveSuccess & {
+        focused: boolean;
+    }>;
+    dblClick(ref: string, opts?: ResolveOptions): Promise<ResolveSuccess>;
+    private readCheckableState;
+    setChecked(ref: string, checked: boolean, opts?: ResolveOptions): Promise<SetCheckedResult>;
+    private setFileInputBySelector;
+    uploadFiles(ref: string, files: string[], opts?: ResolveOptions): Promise<UploadFilesResult>;
+    drag(source: string, target: string, opts?: {
+        from?: ResolveOptions;
+        to?: ResolveOptions;
+    }): Promise<DragResult>;
+    fillText(ref: string, text: string, opts?: ResolveOptions): Promise<FillTextResult>;
+    pressKey(key: string): Promise<void>;
+    scrollTo(ref: string, opts?: ResolveOptions): Promise<unknown>;
+    getFormState(): Promise<Record<string, unknown>>;
+    scroll(direction?: string, amount?: number): Promise<void>;
+    autoScroll(options?: {
+        times?: number;
+        delayMs?: number;
+    }): Promise<void>;
+    networkRequests(includeStatic?: boolean): Promise<unknown[]>;
+    consoleMessages(_level?: string): Promise<unknown[]>;
+    wait(options: number | WaitOptions): Promise<void>;
+    snapshot(opts?: SnapshotOptions): Promise<unknown>;
+    private collectAxSnapshotTrees;
+    getCurrentUrl(): Promise<string | null>;
+    installInterceptor(pattern: string): Promise<void>;
+    getInterceptedRequests(): Promise<unknown[]>;
+    waitForCapture(timeout?: number): Promise<void>;
+    /** Fallback basic snapshot */
+    protected _basicSnapshot(opts?: Pick<SnapshotOptions, 'interactive' | 'compact' | 'maxDepth' | 'raw'>): Promise<unknown>;
+}