@jackwener/opencli 1.7.14 → 1.7.15

package/dist/src/browser/ax-snapshot.js

@@ -0,0 +1,217 @@
+/**
+ * AX-backed browser snapshot prototype.
+ *
+ * This is intentionally additive to the current DOM snapshot. It learns from
+ * agent-browser's accessibility-tree refs without changing default `state`
+ * output until the AX path proves itself on fixtures and real SaaS workflows.
+ */
+const INTERACTIVE_ROLES = new Set([
+    'button',
+    'link',
+    'textbox',
+    'searchbox',
+    'checkbox',
+    'radio',
+    'combobox',
+    'listbox',
+    'menuitem',
+    'menuitemcheckbox',
+    'menuitemradio',
+    'option',
+    'slider',
+    'spinbutton',
+    'switch',
+    'tab',
+    'treeitem',
+]);
+const CONTENT_ROLES = new Set([
+    'article',
+    'cell',
+    'columnheader',
+    'gridcell',
+    'heading',
+    'listitem',
+    'main',
+    'navigation',
+    'region',
+    'rowheader',
+]);
+const STRUCTURAL_ROLES = new Set([
+    'generic',
+    'group',
+    'list',
+    'none',
+    'presentation',
+    'RootWebArea',
+    'WebArea',
+]);
+export function buildAxSnapshot(axTree, opts = {}) {
+    return buildAxSnapshotFromTrees([{ tree: axTree }], opts);
+}
+export function buildAxSnapshotFromTrees(trees, opts = {}) {
+    const lines = ['source: ax', '---'];
+    const refs = new Map();
+    let nextRef = 1;
+    for (const [index, entry] of trees.entries()) {
+        if (index > 0) {
+            const label = entry.frame?.url ? JSON.stringify(entry.frame.url) : JSON.stringify(entry.frame?.frameId ?? `frame:${index}`);
+            lines.push(`frame ${label}:`);
+        }
+        nextRef = renderAxTree(entry.tree, lines, refs, nextRef, {
+            ...opts,
+            frame: entry.frame,
+            baseDepth: index > 0 ? 1 : 0,
+        });
+    }
+    lines.push('---');
+    lines.push(`interactive: ${refs.size}`);
+    return { text: lines.join('\n'), refs };
+}
+function renderAxTree(axTree, lines, refs, nextRef, opts) {
+    const rawNodes = Array.isArray(axTree?.nodes)
+        ? axTree.nodes
+        : [];
+    const nodes = rawNodes.filter((node) => node && !node.ignored);
+    const byId = new Map();
+    const parentIds = new Set();
+    for (const node of nodes) {
+        if (typeof node.nodeId === 'string')
+            byId.set(node.nodeId, node);
+        for (const childId of node.childIds ?? [])
+            parentIds.add(childId);
+    }
+    const roots = nodes.filter((node) => {
+        if (!node.nodeId)
+            return false;
+        const role = axString(node.role);
+        return !parentIds.has(node.nodeId) || role === 'RootWebArea' || role === 'WebArea';
+    });
+    const root = roots[0] ?? nodes[0];
+    const maxDepth = Math.max(1, Math.min(Number(opts.maxDepth) || 50, 200));
+    const roleNameCounts = countRoleNames(nodes);
+    const roleNameSeen = new Map();
+    function render(node, depth) {
+        if (!node || depth > maxDepth)
+            return false;
+        const role = axString(node.role) || 'generic';
+        const name = cleanText(axString(node.name));
+        const value = cleanText(axString(node.value) || propertyValue(node, 'value'));
+        const disabled = propertyValue(node, 'disabled');
+        const checked = propertyValue(node, 'checked');
+        const expanded = propertyValue(node, 'expanded');
+        const selected = propertyValue(node, 'selected');
+        const refEligible = shouldRef(role, name, node.backendDOMNodeId);
+        const shouldShowSelf = refEligible
+            || !!name
+            || !!value
+            || CONTENT_ROLES.has(role)
+            || (!opts.interactiveOnly && !STRUCTURAL_ROLES.has(role));
+        const childStart = lines.length;
+        let hasVisibleChild = false;
+        for (const childId of node.childIds ?? []) {
+            if (render(byId.get(childId), depth + 1))
+                hasVisibleChild = true;
+        }
+        if (!shouldShowSelf && !hasVisibleChild) {
+            lines.length = childStart;
+            return false;
+        }
+        if (shouldShowSelf) {
+            const indent = '  '.repeat(depth);
+            const parts = [];
+            let prefix = '';
+            if (refEligible) {
+                const ref = String(nextRef++);
+                prefix = `[${ref}]`;
+                const key = roleNameKey(role, name);
+                const seen = roleNameSeen.get(key) ?? 0;
+                roleNameSeen.set(key, seen + 1);
+                refs.set(ref, {
+                    ref,
+                    backendNodeId: node.backendDOMNodeId,
+                    role,
+                    name,
+                    ...(roleNameCounts.get(key) > 1 ? { nth: seen } : {}),
+                    ...(opts.frame ? { frame: opts.frame } : {}),
+                });
+            }
+            if (name)
+                parts.push(JSON.stringify(name));
+            if (value && value !== name)
+                parts.push(`value=${JSON.stringify(value)}`);
+            if (checked)
+                parts.push(`checked=${checked}`);
+            if (expanded)
+                parts.push(`expanded=${expanded}`);
+            if (selected)
+                parts.push(`selected=${selected}`);
+            if (disabled === 'true')
+                parts.push('disabled');
+            lines.splice(childStart, 0, `${indent}${prefix}${role}${parts.length ? ` ${parts.join(' ')}` : ''}`);
+        }
+        return true;
+    }
+    render(root, opts.baseDepth);
+    return nextRef;
+}
+export function findAxRefReplacement(axTree, ref) {
+    const nodes = Array.isArray(axTree?.nodes)
+        ? axTree.nodes
+        : [];
+    const targetNth = ref.nth ?? 0;
+    let seen = 0;
+    for (const node of nodes) {
+        if (!node || node.ignored)
+            continue;
+        const role = axString(node.role);
+        const name = cleanText(axString(node.name));
+        if (role !== ref.role || name !== ref.name)
+            continue;
+        if (seen === targetNth) {
+            if (typeof node.backendDOMNodeId !== 'number')
+                return null;
+            return { ...ref, backendNodeId: node.backendDOMNodeId };
+        }
+        seen++;
+    }
+    return null;
+}
+function shouldRef(role, name, backendNodeId) {
+    if (typeof backendNodeId !== 'number')
+        return false;
+    if (INTERACTIVE_ROLES.has(role))
+        return true;
+    return CONTENT_ROLES.has(role) && !!name;
+}
+function countRoleNames(nodes) {
+    const counts = new Map();
+    for (const node of nodes) {
+        if (!node || node.ignored)
+            continue;
+        const role = axString(node.role);
+        const name = cleanText(axString(node.name));
+        if (!shouldRef(role, name, node.backendDOMNodeId))
+            continue;
+        const key = roleNameKey(role, name);
+        counts.set(key, (counts.get(key) ?? 0) + 1);
+    }
+    return counts;
+}
+function roleNameKey(role, name) {
+    return `${role}\u0000${name}`;
+}
+function axString(value) {
+    const raw = value?.value;
+    if (typeof raw === 'string')
+        return raw;
+    if (typeof raw === 'number' || typeof raw === 'boolean')
+        return String(raw);
+    return '';
+}
+function propertyValue(node, name) {
+    const prop = node.properties?.find((candidate) => candidate.name === name);
+    return axString(prop?.value);
+}
+function cleanText(value) {
+    return value.replace(/\s+/g, ' ').trim().slice(0, 160);
+}

package/dist/src/browser/ax-snapshot.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/browser/ax-snapshot.test.js

@@ -0,0 +1,91 @@
+import { describe, expect, it } from 'vitest';
+import { buildAxSnapshot, buildAxSnapshotFromTrees, findAxRefReplacement } from './ax-snapshot.js';
+describe('AX snapshot prototype', () => {
+    it('builds compact refs from Accessibility.getFullAXTree output', () => {
+        const result = buildAxSnapshot({
+            nodes: [
+                { nodeId: '1', role: { value: 'RootWebArea' }, name: { value: 'Example' }, childIds: ['2', '3'] },
+                { nodeId: '2', role: { value: 'heading' }, name: { value: 'Expenses' }, backendDOMNodeId: 20 },
+                {
+                    nodeId: '3',
+                    role: { value: 'combobox' },
+                    name: { value: 'Category' },
+                    backendDOMNodeId: 30,
+                    properties: [{ name: 'expanded', value: { value: false } }],
+                },
+            ],
+        });
+        expect(result.text).toContain('source: ax');
+        expect(result.text).toContain('[1]heading "Expenses"');
+        expect(result.text).toContain('[2]combobox "Category" expanded=false');
+        expect(result.refs.get('2')).toEqual({
+            ref: '2',
+            backendNodeId: 30,
+            role: 'combobox',
+            name: 'Category',
+        });
+    });
+    it('tracks nth only for duplicate role/name pairs', () => {
+        const result = buildAxSnapshot({
+            nodes: [
+                { nodeId: '1', role: { value: 'RootWebArea' }, childIds: ['2', '3'] },
+                { nodeId: '2', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 10 },
+                { nodeId: '3', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 11 },
+            ],
+        });
+        expect(result.refs.get('1')).toMatchObject({ role: 'button', name: 'Save', nth: 0 });
+        expect(result.refs.get('2')).toMatchObject({ role: 'button', name: 'Save', nth: 1 });
+    });
+    it('finds stale ref replacements by role/name/nth', () => {
+        const replacement = findAxRefReplacement({
+            nodes: [
+                { nodeId: '1', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 10 },
+                { nodeId: '2', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 42 },
+            ],
+        }, {
+            ref: '2',
+            role: 'button',
+            name: 'Save',
+            backendNodeId: 11,
+            nth: 1,
+        });
+        expect(replacement).toMatchObject({ ref: '2', backendNodeId: 42, role: 'button', name: 'Save', nth: 1 });
+    });
+    it('combines frame AX trees while keeping ref metadata frame-scoped', () => {
+        const result = buildAxSnapshotFromTrees([
+            {
+                tree: {
+                    nodes: [
+                        { nodeId: '1', role: { value: 'RootWebArea' }, childIds: ['2'] },
+                        { nodeId: '2', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 10 },
+                    ],
+                },
+            },
+            {
+                frame: { frameId: 'frame-1', url: 'https://app.example/embed' },
+                tree: {
+                    nodes: [
+                        { nodeId: '1', role: { value: 'RootWebArea' }, childIds: ['2'] },
+                        { nodeId: '2', role: { value: 'button' }, name: { value: 'Save' }, backendDOMNodeId: 20 },
+                    ],
+                },
+            },
+        ]);
+        expect(result.text).toContain('[1]button "Save"');
+        expect(result.text).toContain('frame "https://app.example/embed":');
+        expect(result.text).toContain('  [2]button "Save"');
+        expect(result.refs.get('1')).toEqual({
+            ref: '1',
+            backendNodeId: 10,
+            role: 'button',
+            name: 'Save',
+        });
+        expect(result.refs.get('2')).toEqual({
+            ref: '2',
+            backendNodeId: 20,
+            role: 'button',
+            name: 'Save',
+            frame: { frameId: 'frame-1', url: 'https://app.example/embed' },
+        });
+    });
+});

package/dist/src/browser/base-page.d.ts

@@ -27,11 +27,34 @@ export interface FillTextResult extends ResolveSuccess {
     length: number;
     mode?: 'input' | 'textarea' | 'contenteditable';
 }
+export interface SetCheckedResult extends ResolveSuccess {
+    checked: boolean;
+    changed: boolean;
+    kind?: string;
+}
+export interface UploadFilesResult extends ResolveSuccess {
+    uploaded: boolean;
+    files: number;
+    file_names: string[];
+    target: string;
+    multiple?: boolean;
+    accept?: string;
+}
+export interface DragResult {
+    dragged: boolean;
+    source: string;
+    target: string;
+    source_matches_n: number;
+    target_matches_n: number;
+    source_match_level: TargetMatchLevel;
+    target_match_level: TargetMatchLevel;
+}
 export declare abstract class BasePage implements IPage {
     protected _lastUrl: string | null;
     /** Cached previous snapshot hashes for incremental diff marking */
     private _prevSnapshotHashes;
     private _cdpTargetMarkerSeq;
+    private _axRefs;
     abstract goto(url: string, options?: {
         waitUntil?: 'load' | 'none';
         settleMs?: number;
@@ -53,15 +76,29 @@ export declare abstract class BasePage implements IPage {
         url?: string;
     }): Promise<BrowserCookie[]>;
     abstract screenshot(options?: ScreenshotOptions): Promise<string>;
+    annotatedScreenshot(options?: ScreenshotOptions): Promise<string>;
     abstract tabs(): Promise<unknown[]>;
     abstract selectTab(target: number | string): Promise<void>;
     click(ref: string, opts?: ResolveOptions): Promise<ResolveSuccess>;
     /** Uses native CDP click support when the concrete page exposes it. */
     protected tryNativeClick(x: number, y: number): Promise<boolean>;
+    protected tryNativeMouseMove(x: number, y: number): Promise<boolean>;
+    protected tryNativeDoubleClick(x: number, y: number): Promise<boolean>;
+    protected tryNativeDrag(from: {
+        x: number;
+        y: number;
+    }, to: {
+        x: number;
+        y: number;
+    }): Promise<boolean>;
+    protected tryClickAxRef(ref: string): Promise<ResolveSuccess | null>;
+    private resolveAxRefPoint;
+    private axBoxCenter;
     /** Uses native CDP text insertion when the concrete page exposes it. */
     protected tryNativeType(text: string): Promise<boolean>;
     /** Uses native CDP key events when the concrete page exposes them. */
     protected tryNativeKeyPress(key: string, modifiers: string[]): Promise<boolean>;
+    protected isResolvedFocused(): Promise<boolean>;
     /**
      * Run a DOM-domain CDP command against `window.__resolved`.
      *
@@ -71,6 +108,19 @@ export declare abstract class BasePage implements IPage {
      */
     protected tryCdpOnResolvedElement(method: 'DOM.focus' | 'DOM.scrollIntoViewIfNeeded'): Promise<boolean>;
     typeText(ref: string, text: string, opts?: ResolveOptions): Promise<ResolveSuccess>;
+    hover(ref: string, opts?: ResolveOptions): Promise<ResolveSuccess>;
+    focus(ref: string, opts?: ResolveOptions): Promise<ResolveSuccess & {
+        focused: boolean;
+    }>;
+    dblClick(ref: string, opts?: ResolveOptions): Promise<ResolveSuccess>;
+    private readCheckableState;
+    setChecked(ref: string, checked: boolean, opts?: ResolveOptions): Promise<SetCheckedResult>;
+    private setFileInputBySelector;
+    uploadFiles(ref: string, files: string[], opts?: ResolveOptions): Promise<UploadFilesResult>;
+    drag(source: string, target: string, opts?: {
+        from?: ResolveOptions;
+        to?: ResolveOptions;
+    }): Promise<DragResult>;
     fillText(ref: string, text: string, opts?: ResolveOptions): Promise<FillTextResult>;
     pressKey(key: string): Promise<void>;
     scrollTo(ref: string, opts?: ResolveOptions): Promise<unknown>;
@@ -84,6 +134,7 @@ export declare abstract class BasePage implements IPage {
     consoleMessages(_level?: string): Promise<unknown[]>;
     wait(options: number | WaitOptions): Promise<void>;
     snapshot(opts?: SnapshotOptions): Promise<unknown>;
+    private collectAxSnapshotTrees;
     getCurrentUrl(): Promise<string | null>;
     installInterceptor(pattern: string): Promise<void>;
     getInterceptedRequests(): Promise<unknown[]>;