agent-browser 0.1.2 → 0.2.0

package/dist/snapshot.js

@@ -0,0 +1,298 @@
+/**
+ * Enhanced snapshot with element refs for deterministic element selection.
+ *
+ * This module generates accessibility snapshots with embedded refs that can be
+ * used to click/fill/interact with elements without re-querying the DOM.
+ *
+ * Example output:
+ *   - heading "Example Domain" [ref=e1] [level=1]
+ *   - paragraph: Some text content
+ *   - button "Submit" [ref=e2]
+ *   - textbox "Email" [ref=e3]
+ *
+ * Usage:
+ *   agent-browser snapshot              # Full snapshot
+ *   agent-browser snapshot -i           # Interactive elements only
+ *   agent-browser snapshot --depth 3    # Limit depth
+ *   agent-browser click @e2             # Click element by ref
+ */
+// Counter for generating refs
+let refCounter = 0;
+/**
+ * Reset ref counter (call at start of each snapshot)
+ */
+export function resetRefs() {
+    refCounter = 0;
+}
+/**
+ * Generate next ref ID
+ */
+function nextRef() {
+    return `e${++refCounter}`;
+}
+/**
+ * Roles that are interactive and should get refs
+ */
+const INTERACTIVE_ROLES = new Set([
+    'button',
+    'link',
+    'textbox',
+    'checkbox',
+    'radio',
+    'combobox',
+    'listbox',
+    'menuitem',
+    'menuitemcheckbox',
+    'menuitemradio',
+    'option',
+    'searchbox',
+    'slider',
+    'spinbutton',
+    'switch',
+    'tab',
+    'treeitem',
+]);
+/**
+ * Roles that provide structure/context (get refs for text extraction)
+ */
+const CONTENT_ROLES = new Set([
+    'heading',
+    'cell',
+    'gridcell',
+    'columnheader',
+    'rowheader',
+    'listitem',
+    'article',
+    'region',
+    'main',
+    'navigation',
+]);
+/**
+ * Roles that are purely structural (can be filtered in compact mode)
+ */
+const STRUCTURAL_ROLES = new Set([
+    'generic',
+    'group',
+    'list',
+    'table',
+    'row',
+    'rowgroup',
+    'grid',
+    'treegrid',
+    'menu',
+    'menubar',
+    'toolbar',
+    'tablist',
+    'tree',
+    'directory',
+    'document',
+    'application',
+    'presentation',
+    'none',
+]);
+/**
+ * Build a selector string for storing in ref map
+ */
+function buildSelector(role, name) {
+    if (name) {
+        const escapedName = name.replace(/"/g, '\\"');
+        return `getByRole('${role}', { name: "${escapedName}" })`;
+    }
+    return `getByRole('${role}')`;
+}
+/**
+ * Get enhanced snapshot with refs and optional filtering
+ */
+export async function getEnhancedSnapshot(page, options = {}) {
+    resetRefs();
+    const refs = {};
+    // Get ARIA snapshot from Playwright
+    const locator = options.selector ? page.locator(options.selector) : page.locator(':root');
+    const ariaTree = await locator.ariaSnapshot();
+    if (!ariaTree) {
+        return {
+            tree: '(empty)',
+            refs: {},
+        };
+    }
+    // Parse and enhance the ARIA tree
+    const enhancedTree = processAriaTree(ariaTree, refs, options);
+    return { tree: enhancedTree, refs };
+}
+/**
+ * Process ARIA snapshot: add refs and apply filters
+ */
+function processAriaTree(ariaTree, refs, options) {
+    const lines = ariaTree.split('\n');
+    const result = [];
+    // For interactive-only mode, we collect just interactive elements
+    if (options.interactive) {
+        for (const line of lines) {
+            const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
+            if (!match)
+                continue;
+            const [, , role, name, suffix] = match;
+            const roleLower = role.toLowerCase();
+            if (INTERACTIVE_ROLES.has(roleLower)) {
+                const ref = nextRef();
+                refs[ref] = {
+                    selector: buildSelector(roleLower, name),
+                    role: roleLower,
+                    name,
+                };
+                let enhanced = `- ${role}`;
+                if (name)
+                    enhanced += ` "${name}"`;
+                enhanced += ` [ref=${ref}]`;
+                if (suffix && suffix.includes('['))
+                    enhanced += suffix;
+                result.push(enhanced);
+            }
+        }
+        return result.join('\n') || '(no interactive elements)';
+    }
+    // Normal processing with depth/compact filters
+    for (const line of lines) {
+        const processed = processLine(line, refs, options);
+        if (processed !== null) {
+            result.push(processed);
+        }
+    }
+    // If compact mode, remove empty structural elements
+    if (options.compact) {
+        return compactTree(result.join('\n'));
+    }
+    return result.join('\n');
+}
+/**
+ * Get indentation level (number of spaces / 2)
+ */
+function getIndentLevel(line) {
+    const match = line.match(/^(\s*)/);
+    return match ? Math.floor(match[1].length / 2) : 0;
+}
+/**
+ * Process a single line: add ref if needed, filter if requested
+ */
+function processLine(line, refs, options) {
+    const depth = getIndentLevel(line);
+    // Check max depth
+    if (options.maxDepth !== undefined && depth > options.maxDepth) {
+        return null;
+    }
+    // Match lines like:
+    //   - button "Submit"
+    //   - heading "Title" [level=1]
+    //   - link "Click me":
+    const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
+    if (!match) {
+        // Metadata lines (like /url:) or text content
+        if (options.interactive) {
+            // In interactive mode, only keep metadata under interactive elements
+            return null;
+        }
+        return line;
+    }
+    const [, prefix, role, name, suffix] = match;
+    const roleLower = role.toLowerCase();
+    // Skip metadata lines (like /url:)
+    if (role.startsWith('/')) {
+        return line;
+    }
+    const isInteractive = INTERACTIVE_ROLES.has(roleLower);
+    const isContent = CONTENT_ROLES.has(roleLower);
+    const isStructural = STRUCTURAL_ROLES.has(roleLower);
+    // In interactive-only mode, filter non-interactive elements
+    if (options.interactive && !isInteractive) {
+        return null;
+    }
+    // In compact mode, skip unnamed structural elements
+    if (options.compact && isStructural && !name) {
+        return null;
+    }
+    // Add ref for interactive or named content elements
+    const shouldHaveRef = isInteractive || (isContent && name);
+    if (shouldHaveRef) {
+        const ref = nextRef();
+        refs[ref] = {
+            selector: buildSelector(roleLower, name),
+            role: roleLower,
+            name,
+        };
+        // Build enhanced line with ref
+        let enhanced = `${prefix}${role}`;
+        if (name)
+            enhanced += ` "${name}"`;
+        enhanced += ` [ref=${ref}]`;
+        if (suffix)
+            enhanced += suffix;
+        return enhanced;
+    }
+    return line;
+}
+/**
+ * Remove empty structural branches in compact mode
+ */
+function compactTree(tree) {
+    const lines = tree.split('\n');
+    const result = [];
+    // Simple pass: keep lines that have content or refs
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        // Always keep lines with refs
+        if (line.includes('[ref=')) {
+            result.push(line);
+            continue;
+        }
+        // Keep lines with text content (after :)
+        if (line.includes(':') && !line.endsWith(':')) {
+            result.push(line);
+            continue;
+        }
+        // Check if this structural element has children with refs
+        const currentIndent = getIndentLevel(line);
+        let hasRelevantChildren = false;
+        for (let j = i + 1; j < lines.length; j++) {
+            const childIndent = getIndentLevel(lines[j]);
+            if (childIndent <= currentIndent)
+                break;
+            if (lines[j].includes('[ref=')) {
+                hasRelevantChildren = true;
+                break;
+            }
+        }
+        if (hasRelevantChildren) {
+            result.push(line);
+        }
+    }
+    return result.join('\n');
+}
+/**
+ * Parse a ref from command argument (e.g., "@e1" -> "e1")
+ */
+export function parseRef(arg) {
+    if (arg.startsWith('@')) {
+        return arg.slice(1);
+    }
+    if (arg.startsWith('ref=')) {
+        return arg.slice(4);
+    }
+    if (/^e\d+$/.test(arg)) {
+        return arg;
+    }
+    return null;
+}
+/**
+ * Get snapshot statistics
+ */
+export function getSnapshotStats(tree, refs) {
+    const interactive = Object.values(refs).filter(r => INTERACTIVE_ROLES.has(r.role)).length;
+    return {
+        lines: tree.split('\n').length,
+        chars: tree.length,
+        tokens: Math.ceil(tree.length / 4),
+        refs: Object.keys(refs).length,
+        interactive,
+    };
+}
+//# sourceMappingURL=snapshot.js.map

package/dist/snapshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../src/snapshot.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AA4BH,8BAA8B;AAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;AAEnB;;GAEG;AACH,MAAM,UAAU,SAAS;IACvB,UAAU,GAAG,CAAC,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,OAAO;IACd,OAAO,IAAI,EAAE,UAAU,EAAE,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,QAAQ;IACR,MAAM;IACN,SAAS;IACT,UAAU;IACV,OAAO;IACP,UAAU;IACV,SAAS;IACT,UAAU;IACV,kBAAkB;IAClB,eAAe;IACf,QAAQ;IACR,WAAW;IACX,QAAQ;IACR,YAAY;IACZ,QAAQ;IACR,KAAK;IACL,UAAU;CACX,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,SAAS;IACT,MAAM;IACN,UAAU;IACV,cAAc;IACd,WAAW;IACX,UAAU;IACV,SAAS;IACT,QAAQ;IACR,MAAM;IACN,YAAY;CACb,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC;IAC/B,SAAS;IACT,OAAO;IACP,MAAM;IACN,OAAO;IACP,KAAK;IACL,UAAU;IACV,MAAM;IACN,UAAU;IACV,MAAM;IACN,SAAS;IACT,SAAS;IACT,SAAS;IACT,MAAM;IACN,WAAW;IACX,UAAU;IACV,aAAa;IACb,cAAc;IACd,MAAM;CACP,CAAC,CAAC;AAEH;;GAEG;AACH,SAAS,aAAa,CAAC,IAAY,EAAE,IAAa;IAChD,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC9C,OAAO,cAAc,IAAI,eAAe,WAAW,MAAM,CAAC;IAC5D,CAAC;IACD,OAAO,cAAc,IAAI,IAAI,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAU,EACV,UAA2B,EAAE;IAE7B,SAAS,EAAE,CAAC;IACZ,MAAM,IAAI,GAAW,EAAE,CAAC;IAExB,oCAAoC;IACpC,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1F,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;IAE9C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO;YACL,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,EAAE;SACT,CAAC;IACJ,CAAC;IAED,kCAAkC;IAClC,MAAM,YAAY,GAAG,eAAe,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,QAAgB,EAAE,IAAY,EAAE,OAAwB;IAC/E,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,kEAAkE;IAClE,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAClE,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,MAAM,CAAC,EAAE,AAAD,EAAG,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC;YACvC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YAErC,IAAI,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrC,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC;gBACtB,IAAI,CAAC,GAAG,CAAC,GAAG;oBACV,QAAQ,EAAE,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC;oBACxC,IAAI,EAAE,SAAS;oBACf,IAAI;iBACL,CAAC;gBAEF,IAAI,QAAQ,GAAG,KAAK,IAAI,EAAE,CAAC;gBAC3B,IAAI,IAAI;oBAAE,QAAQ,IAAI,KAAK,IAAI,GAAG,CAAC;gBACnC,QAAQ,IAAI,SAAS,GAAG,GAAG,CAAC;gBAC5B,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;oBAAE,QAAQ,IAAI,MAAM,CAAC;gBAEvD,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,2BAA2B,CAAC;IAC1D,CAAC;IAED,+CAA+C;IAC/C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QACnD,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,oDAAoD;IACpD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACnC,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAClB,IAAY,EACZ,IAAY,EACZ,OAAwB;IAExB,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAEnC,kBAAkB;IAClB,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC/D,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oBAAoB;IACpB,sBAAsB;IACtB,gCAAgC;IAChC,uBAAuB;IACvB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAElE,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,8CAA8C;QAC9C,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;YACxB,qEAAqE;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC;IAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAErC,mCAAmC;IACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,aAAa,GAAG,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAErD,4DAA4D;IAC5D,IAAI,OAAO,CAAC,WAAW,IAAI,CAAC,aAAa,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oDAAoD;IACpD,IAAI,OAAO,CAAC,OAAO,IAAI,YAAY,IAAI,CAAC,IAAI,EAAE,CAAC;QAC7C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oDAAoD;IACpD,MAAM,aAAa,GAAG,aAAa,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;IAE3D,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,OAAO,EAAE,CAAC;QAEtB,IAAI,CAAC,GAAG,CAAC,GAAG;YACV,QAAQ,EAAE,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC;YACxC,IAAI,EAAE,SAAS;YACf,IAAI;SACL,CAAC;QAEF,+BAA+B;QAC/B,IAAI,QAAQ,GAAG,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC;QAClC,IAAI,IAAI;YAAE,QAAQ,IAAI,KAAK,IAAI,GAAG,CAAC;QACnC,QAAQ,IAAI,SAAS,GAAG,GAAG,CAAC;QAC5B,IAAI,MAAM;YAAE,QAAQ,IAAI,MAAM,CAAC;QAE/B,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,oDAAoD;IACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEtB,8BAA8B;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QAED,yCAAyC;QACzC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QAED,0DAA0D;QAC1D,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,mBAAmB,GAAG,KAAK,CAAC;QAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,WAAW,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7C,IAAI,WAAW,IAAI,aAAa;gBAAE,MAAM;YACxC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/B,mBAAmB,GAAG,IAAI,CAAC;gBAC3B,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,mBAAmB,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IACD,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,IAAY;IAOzD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACjD,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAC9B,CAAC,MAAM,CAAC;IAET,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;QAC9B,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM;QAC9B,WAAW;KACZ,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-browser",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Headless browser automation CLI for AI agents",
   "type": "module",
   "main": "dist/index.js",
@@ -8,7 +8,8 @@
     "agent-browser": "./bin/agent-browser"
   },
   "scripts": {
-    "build": "tsc",
+    "build": "tsc && bun build ./src/cli-light.ts --compile --outfile ./bin/agent-browser",
+    "build:node": "tsc",
     "start": "node dist/index.js",
     "dev": "tsx src/index.ts",
     "typecheck": "tsc --noEmit",
@@ -29,12 +30,12 @@
   "author": "",
   "license": "Apache-2.0",
   "dependencies": {
-    "playwright-core": "^1.40.0",
+    "playwright-core": "^1.57.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {
     "@types/node": "^20.10.0",
-    "playwright": "^1.40.0",
+    "playwright": "^1.57.0",
     "prettier": "^3.7.4",
     "tsx": "^4.6.0",
     "typescript": "^5.3.0",

package/src/actions.ts

@@ -108,6 +108,7 @@ import { successResponse, errorResponse } from './protocol.js';
 // Snapshot response type
 interface SnapshotData {
   snapshot: string;
+  refs?: Record<string, { role: string; name?: string }>;
 }
 
 /**
@@ -380,8 +381,10 @@ async function handleNavigate(
 }
 
 async function handleClick(command: ClickCommand, browser: BrowserManager): Promise<Response> {
-  const page = browser.getPage();
-  await page.click(command.selector, {
+  // Support both refs (@e1) and regular selectors
+  const locator = browser.getLocator(command.selector);
+  
+  await locator.click({
     button: command.button,
     clickCount: command.clickCount,
     delay: command.delay,
@@ -391,13 +394,13 @@ async function handleClick(command: ClickCommand, browser: BrowserManager): Prom
 }
 
 async function handleType(command: TypeCommand, browser: BrowserManager): Promise<Response> {
-  const page = browser.getPage();
+  const locator = browser.getLocator(command.selector);
 
   if (command.clear) {
-    await page.fill(command.selector, '');
+    await locator.fill('');
   }
 
-  await page.type(command.selector, command.text, {
+  await locator.pressSequentially(command.text, {
     delay: command.delay,
   });
 
@@ -446,15 +449,26 @@ async function handleScreenshot(
 }
 
 async function handleSnapshot(
-  command: Command & { action: 'snapshot' },
+  command: Command & { action: 'snapshot'; interactive?: boolean; maxDepth?: number; compact?: boolean; selector?: string },
   browser: BrowserManager
 ): Promise<Response<SnapshotData>> {
-  const page = browser.getPage();
-  // Use ariaSnapshot which returns a string representation of the accessibility tree
-  const snapshot = await page.locator(':root').ariaSnapshot();
+  // Use enhanced snapshot with refs and optional filtering
+  const { tree, refs } = await browser.getSnapshot({
+    interactive: command.interactive,
+    maxDepth: command.maxDepth,
+    compact: command.compact,
+    selector: command.selector,
+  });
+
+  // Simplify refs for output (just role and name)
+  const simpleRefs: Record<string, { role: string; name?: string }> = {};
+  for (const [ref, data] of Object.entries(refs)) {
+    simpleRefs[ref] = { role: data.role, name: data.name };
+  }
 
   return successResponse(command.id, {
-    snapshot: snapshot ?? 'Empty page',
+    snapshot: tree || 'Empty page',
+    refs: Object.keys(simpleRefs).length > 0 ? simpleRefs : undefined,
   });
 }
 
@@ -533,17 +547,17 @@ async function handleScroll(command: ScrollCommand, browser: BrowserManager): Pr
 }
 
 async function handleSelect(command: SelectCommand, browser: BrowserManager): Promise<Response> {
-  const page = browser.getPage();
+  const locator = browser.getLocator(command.selector);
   const values = Array.isArray(command.values) ? command.values : [command.values];
 
-  await page.selectOption(command.selector, values);
+  await locator.selectOption(values);
 
   return successResponse(command.id, { selected: values });
 }
 
 async function handleHover(command: HoverCommand, browser: BrowserManager): Promise<Response> {
-  const page = browser.getPage();
-  await page.hover(command.selector);
+  const locator = browser.getLocator(command.selector);
+  await locator.hover();
 
   return successResponse(command.id, { hovered: true });
 }
@@ -622,27 +636,27 @@ async function handleWindowNew(
 // New handlers for enhanced Playwright parity
 
 async function handleFill(command: FillCommand, browser: BrowserManager): Promise<Response> {
-  const frame = browser.getFrame();
-  await frame.fill(command.selector, command.value);
+  const locator = browser.getLocator(command.selector);
+  await locator.fill(command.value);
   return successResponse(command.id, { filled: true });
 }
 
 async function handleCheck(command: CheckCommand, browser: BrowserManager): Promise<Response> {
-  const frame = browser.getFrame();
-  await frame.check(command.selector);
+  const locator = browser.getLocator(command.selector);
+  await locator.check();
   return successResponse(command.id, { checked: true });
 }
 
 async function handleUncheck(command: UncheckCommand, browser: BrowserManager): Promise<Response> {
-  const frame = browser.getFrame();
-  await frame.uncheck(command.selector);
+  const locator = browser.getLocator(command.selector);
+  await locator.uncheck();
   return successResponse(command.id, { unchecked: true });
 }
 
 async function handleUpload(command: UploadCommand, browser: BrowserManager): Promise<Response> {
-  const frame = browser.getFrame();
+  const locator = browser.getLocator(command.selector);
   const files = Array.isArray(command.files) ? command.files : [command.files];
-  await frame.setInputFiles(command.selector, files);
+  await locator.setInputFiles(files);
   return successResponse(command.id, { uploaded: files });
 }
 
@@ -650,14 +664,14 @@ async function handleDoubleClick(
   command: DoubleClickCommand,
   browser: BrowserManager
 ): Promise<Response> {
-  const frame = browser.getFrame();
-  await frame.dblclick(command.selector);
+  const locator = browser.getLocator(command.selector);
+  await locator.dblclick();
   return successResponse(command.id, { clicked: true });
 }
 
 async function handleFocus(command: FocusCommand, browser: BrowserManager): Promise<Response> {
-  const frame = browser.getFrame();
-  await frame.focus(command.selector);
+  const locator = browser.getLocator(command.selector);
+  await locator.focus();
   return successResponse(command.id, { focused: true });
 }
 
@@ -1017,14 +1031,14 @@ async function handleGetAttribute(
   command: GetAttributeCommand,
   browser: BrowserManager
 ): Promise<Response> {
-  const page = browser.getPage();
-  const value = await page.getAttribute(command.selector, command.attribute);
+  const locator = browser.getLocator(command.selector);
+  const value = await locator.getAttribute(command.attribute);
   return successResponse(command.id, { attribute: command.attribute, value });
 }
 
 async function handleGetText(command: GetTextCommand, browser: BrowserManager): Promise<Response> {
-  const page = browser.getPage();
-  const text = await page.textContent(command.selector);
+  const locator = browser.getLocator(command.selector);
+  const text = await locator.textContent();
   return successResponse(command.id, { text });
 }
 
@@ -1032,8 +1046,8 @@ async function handleIsVisible(
   command: IsVisibleCommand,
   browser: BrowserManager
 ): Promise<Response> {
-  const page = browser.getPage();
-  const visible = await page.isVisible(command.selector);
+  const locator = browser.getLocator(command.selector);
+  const visible = await locator.isVisible();
   return successResponse(command.id, { visible });
 }
 
@@ -1041,8 +1055,8 @@ async function handleIsEnabled(
   command: IsEnabledCommand,
   browser: BrowserManager
 ): Promise<Response> {
-  const page = browser.getPage();
-  const enabled = await page.isEnabled(command.selector);
+  const locator = browser.getLocator(command.selector);
+  const enabled = await locator.isEnabled();
   return successResponse(command.id, { enabled });
 }
 
@@ -1050,8 +1064,8 @@ async function handleIsChecked(
   command: IsCheckedCommand,
   browser: BrowserManager
 ): Promise<Response> {
-  const page = browser.getPage();
-  const checked = await page.isChecked(command.selector);
+  const locator = browser.getLocator(command.selector);
+  const checked = await locator.isChecked();
   return successResponse(command.id, { checked });
 }
 

package/src/browser.ts

@@ -10,8 +10,10 @@ import {
   type Dialog,
   type Request,
   type Route,
+  type Locator,
 } from 'playwright-core';
 import type { LaunchCommand } from './types.js';
+import { type RefMap, type EnhancedSnapshot, getEnhancedSnapshot, parseRef } from './snapshot.js';
 
 interface TrackedRequest {
   url: string;
@@ -47,6 +49,8 @@ export class BrowserManager {
   private consoleMessages: ConsoleMessage[] = [];
   private pageErrors: PageError[] = [];
   private isRecordingHar: boolean = false;
+  private refMap: RefMap = {};
+  private lastSnapshot: string = '';
 
   /**
    * Check if browser is launched
@@ -55,6 +59,70 @@ export class BrowserManager {
     return this.browser !== null;
   }
 
+  /**
+   * Get enhanced snapshot with refs and cache the ref map
+   */
+  async getSnapshot(options?: {
+    interactive?: boolean;
+    maxDepth?: number;
+    compact?: boolean;
+    selector?: string;
+  }): Promise<EnhancedSnapshot> {
+    const page = this.getPage();
+    const snapshot = await getEnhancedSnapshot(page, options);
+    this.refMap = snapshot.refs;
+    this.lastSnapshot = snapshot.tree;
+    return snapshot;
+  }
+
+  /**
+   * Get the cached ref map from last snapshot
+   */
+  getRefMap(): RefMap {
+    return this.refMap;
+  }
+
+  /**
+   * Get a locator from a ref (e.g., "e1", "@e1", "ref=e1")
+   * Returns null if ref doesn't exist or is invalid
+   */
+  getLocatorFromRef(refArg: string): Locator | null {
+    const ref = parseRef(refArg);
+    if (!ref) return null;
+
+    const refData = this.refMap[ref];
+    if (!refData) return null;
+
+    const page = this.getPage();
+    
+    // Parse the selector and create locator
+    if (refData.name) {
+      return page.getByRole(refData.role as any, { name: refData.name });
+    } else {
+      return page.getByRole(refData.role as any);
+    }
+  }
+
+  /**
+   * Check if a selector looks like a ref
+   */
+  isRef(selector: string): boolean {
+    return parseRef(selector) !== null;
+  }
+
+  /**
+   * Get locator - supports both refs and regular selectors
+   */
+  getLocator(selectorOrRef: string): Locator {
+    // Check if it's a ref first
+    const locator = this.getLocatorFromRef(selectorOrRef);
+    if (locator) return locator;
+
+    // Otherwise treat as regular selector
+    const page = this.getPage();
+    return page.locator(selectorOrRef);
+  }
+
   /**
    * Get the current active page, throws if not launched
    */
@@ -428,11 +496,12 @@ export class BrowserManager {
 
   /**
    * Launch the browser with the specified options
+   * If already launched, this is a no-op (browser stays open)
    */
   async launch(options: LaunchCommand): Promise<void> {
-    // Close existing browser if any
+    // If already launched, don't relaunch
     if (this.browser) {
-      await this.close();
+      return;
     }
 
     // Select browser type
@@ -611,5 +680,7 @@ export class BrowserManager {
     }
 
     this.activePageIndex = 0;
+    this.refMap = {};
+    this.lastSnapshot = '';
   }
 }