@agentrhq/webcmd 0.1.1 → 0.1.2

package/dist/src/browser/dom-helpers.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Shared DOM operation JS generators.
+ *
+ * Used by both Page (daemon mode) and CDPPage (direct CDP mode)
+ * to eliminate code duplication for click, type, press, wait, scroll, etc.
+ */
+/** Generate JS to click an element by ref.
+ *  Returns { status, x, y, w, h } for CDP fallback when JS click fails. */
+export declare function clickJs(ref: string): string;
+/** Generate JS to type text into an element by ref.
+ *  Uses native setter for React compat + execCommand for contenteditable. */
+export declare function typeTextJs(ref: string, text: string): string;
+/** Generate JS to press a keyboard key */
+export declare function pressKeyJs(key: string, modifiers?: string[]): string;
+/** Generate JS to wait for text to appear in the page */
+export declare function waitForTextJs(text: string, timeoutMs: number): string;
+/** Generate JS for scroll */
+export declare function scrollJs(direction: string, amount: number): string;
+/** Generate JS for auto-scroll with lazy-load detection */
+export declare function autoScrollJs(times: number, delayMs: number): string;
+/** Generate JS to read performance resource entries as network requests */
+export declare function networkRequestsJs(includeStatic: boolean): string;
+/**
+ * Generate JS to wait until the DOM stabilizes (no mutations for `quietMs`),
+ * with a hard cap at `maxMs`. Uses MutationObserver in the browser.
+ *
+ * Returns as soon as the page stops changing, avoiding unnecessary fixed waits.
+ * If document.body is not available, falls back to a fixed sleep of maxMs.
+ */
+export declare function waitForDomStableJs(maxMs: number, quietMs: number): string;
+/**
+ * Generate JS to wait until window.__webcmd_xhr has ≥1 captured response.
+ * Polls every 100ms. Resolves 'captured' on success; rejects after maxMs.
+ * Used after installInterceptor() + goto() instead of a fixed sleep.
+ */
+export declare function waitForCaptureJs(maxMs: number): string;
+/**
+ * Generate JS to wait until document.querySelector(selector) returns a match.
+ * Uses MutationObserver for near-instant resolution; falls back to reject after timeoutMs.
+ */
+export declare function waitForSelectorJs(selector: string, timeoutMs: number): string;

package/dist/src/browser/dom-helpers.js

@@ -0,0 +1,239 @@
+/**
+ * Shared DOM operation JS generators.
+ *
+ * Used by both Page (daemon mode) and CDPPage (direct CDP mode)
+ * to eliminate code duplication for click, type, press, wait, scroll, etc.
+ */
+/** Shared element lookup JS fragment (4-strategy resolution) */
+function resolveElementJs(safeRef, selectorSet) {
+    return `
+      const ref = ${safeRef};
+      let el = document.querySelector('[data-webcmd-ref="' + ref + '"]');
+      if (!el) el = document.querySelector('[data-ref="' + ref + '"]');
+      if (!el && ref.match(/^[a-zA-Z#.\\[]/)) {
+        try { el = document.querySelector(ref); } catch {}
+      }
+      if (!el) {
+        const idx = parseInt(ref, 10);
+        if (!isNaN(idx)) {
+          el = document.querySelectorAll('${selectorSet}')[idx];
+        }
+      }`;
+}
+/** Generate JS to click an element by ref.
+ *  Returns { status, x, y, w, h } for CDP fallback when JS click fails. */
+export function clickJs(ref) {
+    const safeRef = JSON.stringify(ref);
+    return `
+    (() => {
+      ${resolveElementJs(safeRef, 'a, button, input, select, textarea, [role="button"], [tabindex]:not([tabindex="-1"])')}
+      if (!el) throw new Error('Element not found: ' + ref);
+      el.scrollIntoView({ behavior: 'instant', block: 'center' });
+      const rect = el.getBoundingClientRect();
+      const x = Math.round(rect.left + rect.width / 2);
+      const y = Math.round(rect.top + rect.height / 2);
+      try {
+        el.click();
+        return { status: 'clicked', x, y, w: Math.round(rect.width), h: Math.round(rect.height) };
+      } catch (e) {
+        return { status: 'js_failed', x, y, w: Math.round(rect.width), h: Math.round(rect.height), error: e.message };
+      }
+    })()
+  `;
+}
+/** Generate JS to type text into an element by ref.
+ *  Uses native setter for React compat + execCommand for contenteditable. */
+export function typeTextJs(ref, text) {
+    const safeRef = JSON.stringify(ref);
+    const safeText = JSON.stringify(text);
+    return `
+    (() => {
+      ${resolveElementJs(safeRef, 'input, textarea, [contenteditable="true"]')}
+      if (!el) throw new Error('Element not found: ' + ref);
+      el.focus();
+      if (el.isContentEditable) {
+        // Select all content + delete, then insert (supports undo, works with rich text editors)
+        const sel = window.getSelection();
+        const range = document.createRange();
+        range.selectNodeContents(el);
+        sel.removeAllRanges();
+        sel.addRange(range);
+        document.execCommand('delete', false);
+        document.execCommand('insertText', false, ${safeText});
+        el.dispatchEvent(new Event('input', { bubbles: true }));
+      } else {
+        // Use native setter for React/framework compatibility (match element type)
+        const proto = el instanceof HTMLTextAreaElement
+          ? HTMLTextAreaElement.prototype
+          : HTMLInputElement.prototype;
+        const nativeSetter = Object.getOwnPropertyDescriptor(proto, 'value')?.set;
+        if (nativeSetter) {
+          nativeSetter.call(el, ${safeText});
+        } else {
+          el.value = ${safeText};
+        }
+        el.dispatchEvent(new Event('input', { bubbles: true }));
+        el.dispatchEvent(new Event('change', { bubbles: true }));
+      }
+      return 'typed';
+    })()
+  `;
+}
+/** Generate JS to press a keyboard key */
+export function pressKeyJs(key, modifiers = []) {
+    const hasCtrl = modifiers.includes('Ctrl') || modifiers.includes('Control');
+    const hasAlt = modifiers.includes('Alt');
+    const hasMeta = modifiers.includes('Meta');
+    const hasShift = modifiers.includes('Shift');
+    return `
+    (() => {
+      const el = document.activeElement || document.body;
+      const init = {
+        key: ${JSON.stringify(key)},
+        bubbles: true,
+        ctrlKey: ${hasCtrl},
+        altKey: ${hasAlt},
+        metaKey: ${hasMeta},
+        shiftKey: ${hasShift},
+      };
+      el.dispatchEvent(new KeyboardEvent('keydown', init));
+      el.dispatchEvent(new KeyboardEvent('keyup', init));
+      return 'pressed';
+    })()
+  `;
+}
+/** Generate JS to wait for text to appear in the page */
+export function waitForTextJs(text, timeoutMs) {
+    return `
+    new Promise((resolve, reject) => {
+      const deadline = Date.now() + ${timeoutMs};
+      const check = () => {
+        if (document.body.innerText.includes(${JSON.stringify(text)})) return resolve('found');
+        if (Date.now() > deadline) return reject(new Error('Text not found: ' + ${JSON.stringify(text)}));
+        setTimeout(check, 200);
+      };
+      check();
+    })
+  `;
+}
+/** Generate JS for scroll */
+export function scrollJs(direction, amount) {
+    const dx = direction === 'left' ? -amount : direction === 'right' ? amount : 0;
+    const dy = direction === 'up' ? -amount : direction === 'down' ? amount : 0;
+    return `window.scrollBy(${dx}, ${dy})`;
+}
+/** Generate JS for auto-scroll with lazy-load detection */
+export function autoScrollJs(times, delayMs) {
+    return `
+    (async () => {
+      if (!document.body) return;
+      for (let i = 0; i < ${times}; i++) {
+        const lastHeight = document.body.scrollHeight;
+        window.scrollTo(0, lastHeight);
+        await new Promise(resolve => {
+          let timeoutId;
+          const observer = new MutationObserver(() => {
+            if (document.body.scrollHeight > lastHeight) {
+              clearTimeout(timeoutId);
+              observer.disconnect();
+              setTimeout(resolve, 100);
+            }
+          });
+          observer.observe(document.body, { childList: true, subtree: true });
+          timeoutId = setTimeout(() => { observer.disconnect(); resolve(null); }, ${delayMs});
+        });
+      }
+    })()
+  `;
+}
+/** Generate JS to read performance resource entries as network requests */
+export function networkRequestsJs(includeStatic) {
+    return `
+    (() => {
+      const entries = performance.getEntriesByType('resource');
+      return entries
+        ${includeStatic ? '' : '.filter(e => !["img", "font", "css", "script"].some(t => e.initiatorType === t))'}
+        .map(e => ({
+          url: e.name,
+          type: e.initiatorType,
+          duration: Math.round(e.duration),
+          size: e.transferSize || 0,
+        }));
+    })()
+  `;
+}
+/**
+ * Generate JS to wait until the DOM stabilizes (no mutations for `quietMs`),
+ * with a hard cap at `maxMs`. Uses MutationObserver in the browser.
+ *
+ * Returns as soon as the page stops changing, avoiding unnecessary fixed waits.
+ * If document.body is not available, falls back to a fixed sleep of maxMs.
+ */
+export function waitForDomStableJs(maxMs, quietMs) {
+    return `
+    new Promise(resolve => {
+      if (!document.body) {
+        setTimeout(() => resolve('nobody'), ${maxMs});
+        return;
+      }
+      let timer = null;
+      let cap = null;
+      const done = (reason) => {
+        clearTimeout(timer);
+        clearTimeout(cap);
+        obs.disconnect();
+        resolve(reason);
+      };
+      const resetQuiet = () => {
+        clearTimeout(timer);
+        timer = setTimeout(() => done('quiet'), ${quietMs});
+      };
+      const obs = new MutationObserver(resetQuiet);
+      obs.observe(document.body, { childList: true, subtree: true, attributes: true });
+      resetQuiet();
+      cap = setTimeout(() => done('capped'), ${maxMs});
+    })
+  `;
+}
+/**
+ * Generate JS to wait until window.__webcmd_xhr has ≥1 captured response.
+ * Polls every 100ms. Resolves 'captured' on success; rejects after maxMs.
+ * Used after installInterceptor() + goto() instead of a fixed sleep.
+ */
+export function waitForCaptureJs(maxMs) {
+    return `
+    new Promise((resolve, reject) => {
+      const deadline = Date.now() + ${maxMs};
+      const check = () => {
+        if ((window.__webcmd_xhr || []).length > 0) return resolve('captured');
+        if (Date.now() > deadline) return reject(new Error('No network capture within ${maxMs / 1000}s'));
+        setTimeout(check, 100);
+      };
+      check();
+    })
+  `;
+}
+/**
+ * Generate JS to wait until document.querySelector(selector) returns a match.
+ * Uses MutationObserver for near-instant resolution; falls back to reject after timeoutMs.
+ */
+export function waitForSelectorJs(selector, timeoutMs) {
+    return `
+    new Promise((resolve, reject) => {
+      const sel = ${JSON.stringify(selector)};
+      if (document.querySelector(sel)) return resolve('found');
+      const cap = setTimeout(() => {
+        obs.disconnect();
+        reject(new Error('Selector not found: ' + sel));
+      }, ${timeoutMs});
+      const obs = new MutationObserver(() => {
+        if (document.querySelector(sel)) {
+          clearTimeout(cap);
+          obs.disconnect();
+          resolve('found');
+        }
+      });
+      obs.observe(document.body || document.documentElement, { childList: true, subtree: true });
+    })
+  `;
+}

package/dist/src/browser/dom-helpers.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/browser/dom-helpers.test.js

@@ -0,0 +1,106 @@
+import { describe, it, expect } from 'vitest';
+import { autoScrollJs, waitForCaptureJs, waitForSelectorJs } from './dom-helpers.js';
+describe('autoScrollJs', () => {
+    it('returns early without error when document.body is null', async () => {
+        const g = globalThis;
+        const origDoc = g.document;
+        g.document = { body: null, documentElement: {} };
+        g.window = g;
+        const code = autoScrollJs(3, 500);
+        // Should resolve without throwing
+        await expect(eval(code)).resolves.not.toThrow();
+        g.document = origDoc;
+        delete g.window;
+    });
+});
+describe('waitForCaptureJs', () => {
+    it('returns a non-empty string', () => {
+        const code = waitForCaptureJs(1000);
+        expect(typeof code).toBe('string');
+        expect(code.length).toBeGreaterThan(0);
+        expect(code).toContain('__webcmd_xhr');
+        expect(code).toContain('resolve');
+        expect(code).toContain('reject');
+    });
+    it('resolves "captured" when __webcmd_xhr is populated before deadline', async () => {
+        const g = globalThis;
+        const captured = [];
+        g.__webcmd_xhr = captured;
+        g.window = g; // stub window for Node eval
+        const code = waitForCaptureJs(1000);
+        const promise = eval(code);
+        captured.push({ data: 'test' });
+        await expect(promise).resolves.toBe('captured');
+        delete g.__webcmd_xhr;
+        delete g.window;
+    });
+    it('rejects when __webcmd_xhr stays empty past deadline', async () => {
+        const g = globalThis;
+        g.__webcmd_xhr = [];
+        g.window = g;
+        const code = waitForCaptureJs(50); // 50ms timeout
+        const promise = eval(code);
+        await expect(promise).rejects.toThrow('No network capture within 0.05s');
+        delete g.__webcmd_xhr;
+        delete g.window;
+    });
+    it('resolves immediately when __webcmd_xhr already has data', async () => {
+        const g = globalThis;
+        g.__webcmd_xhr = [{ data: 'already here' }];
+        g.window = g;
+        const code = waitForCaptureJs(1000);
+        await expect(eval(code)).resolves.toBe('captured');
+        delete g.__webcmd_xhr;
+        delete g.window;
+    });
+});
+describe('waitForSelectorJs', () => {
+    it('returns a non-empty string', () => {
+        const code = waitForSelectorJs('#app', 1000);
+        expect(typeof code).toBe('string');
+        expect(code).toContain('#app');
+        expect(code).toContain('querySelector');
+        expect(code).toContain('MutationObserver');
+    });
+    it('resolves "found" immediately when selector already present', async () => {
+        const g = globalThis;
+        const fakeEl = { tagName: 'DIV' };
+        g.document = { querySelector: (_) => fakeEl };
+        const code = waitForSelectorJs('[data-testid="primaryColumn"]', 1000);
+        await expect(eval(code)).resolves.toBe('found');
+        delete g.document;
+    });
+    it('resolves "found" when selector appears after DOM mutation', async () => {
+        const g = globalThis;
+        let mutationCallback;
+        g.MutationObserver = class {
+            constructor(cb) { mutationCallback = cb; }
+            observe() { }
+            disconnect() { }
+        };
+        let calls = 0;
+        g.document = {
+            querySelector: (_) => (calls++ > 0 ? { tagName: 'DIV' } : null),
+            body: {},
+        };
+        const code = waitForSelectorJs('#app', 1000);
+        const promise = eval(code);
+        mutationCallback(); // simulate DOM mutation
+        await expect(promise).resolves.toBe('found');
+        delete g.document;
+        delete g.MutationObserver;
+    });
+    it('rejects when selector never appears within timeout', async () => {
+        const g = globalThis;
+        g.MutationObserver = class {
+            constructor(_cb) { }
+            observe() { }
+            disconnect() { }
+        };
+        g.document = { querySelector: (_) => null, body: {} };
+        const code = waitForSelectorJs('#missing', 50);
+        await expect(eval(code)).rejects.toThrow('Selector not found: #missing');
+        delete g.document;
+        delete g.MutationObserver;
+    });
+});

package/dist/src/browser/dom-snapshot.d.ts

@@ -0,0 +1,93 @@
+/**
+ * DOM Snapshot Engine — Advanced DOM pruning for LLM consumption.
+ *
+ * Inspired by browser-use's multi-layer pruning pipeline, adapted for webcmd's
+ * Chrome Extension + CDP architecture. Runs entirely in-page via Runtime.evaluate.
+ *
+ * Pipeline:
+ *   1. Walk DOM tree, collect visibility + layout + interactivity signals
+ *   2. Prune invisible, zero-area, non-content elements
+ *   3. SVG & decoration collapse
+ *   4. Shadow DOM traversal
+ *   5. Same-origin iframe content extraction
+ *   6. Bounding-box parent-child dedup (link/button wrapping children)
+ *   7. Paint-order occlusion detection (overlay/modal coverage)
+ *   8. Attribute whitelist filtering
+ *   9. Table-aware serialization (markdown tables)
+ *  10. Token-efficient serialization with interactive indices
+ *  11. data-ref annotation for click/type targeting
+ *  12. Hidden interactive element hints (scroll-to-reveal)
+ *  13. Incremental diff (mark new elements with *)
+ *
+ * Additional tools:
+ *   - scrollToRefJs(ref) — scroll to a data-webcmd-ref element
+ *   - getFormStateJs()  — extract all form fields as structured JSON
+ *
+ * Compound sidecar:
+ *   After the tree, a `compounds:` section lists rich JSON for every
+ *   date/select/file ref — format, full option list (up to cap) with
+ *   `options_total` reflecting the true count, file `accept` + `multiple`.
+ *   This is what the snapshot's inline attr dump cannot express and what
+ *   agents kept blowing turns on.
+ */
+export interface DomSnapshotOptions {
+    /** Extra pixels beyond viewport to include (default 800) */
+    viewportExpand?: number;
+    /** Maximum DOM depth to traverse (default 50) */
+    maxDepth?: number;
+    /** Only emit interactive elements and their landmark ancestors */
+    interactiveOnly?: boolean;
+    /** Maximum text content length per node (default 120) */
+    maxTextLength?: number;
+    /** Include scroll position info on scrollable containers (default true) */
+    includeScrollInfo?: boolean;
+    /** Enable bounding-box parent-child dedup (default true) */
+    bboxDedup?: boolean;
+    /** Traverse Shadow DOM roots (default true) */
+    includeShadowDom?: boolean;
+    /** Extract same-origin iframe content (default true) */
+    includeIframes?: boolean;
+    /** Maximum number of iframes to process (default 5) */
+    maxIframes?: number;
+    /** Enable paint-order occlusion detection (default true) */
+    paintOrderCheck?: boolean;
+    /** Annotate interactive elements with data-webcmd-ref (default true) */
+    annotateRefs?: boolean;
+    /** Report hidden interactive elements outside viewport (default true) */
+    reportHidden?: boolean;
+    /** Filter ad/noise elements (default true) */
+    filterAds?: boolean;
+    /** Serialize tables as markdown (default true) */
+    markdownTables?: boolean;
+    /** Previous snapshot hash set (JSON array of hashes) for diff marking (default null) */
+    previousHashes?: string | null;
+}
+/**
+ * Generate JS to scroll to an element identified by data-webcmd-ref.
+ * Completes the snapshot→action loop: snapshot identifies `[3]<button>`,
+ * caller can then `scrollToRef('3')` to bring it into view.
+ */
+export declare function scrollToRefJs(ref: string): string;
+/**
+ * Generate JS to extract all form field values from the page.
+ * Returns structured JSON: { forms: [{ id, action, fields: [{ tag, type, name, value, ... }] }] }
+ */
+export declare function getFormStateJs(): string;
+/**
+ * Generate JavaScript code that, when evaluated in a page context via CDP
+ * Runtime.evaluate, returns a pruned DOM snapshot string optimised for LLMs.
+ *
+ * The snapshot output format:
+ *   [42]<button type=submit>Search</button>
+ *   |scroll|<div> (0.5↑ 3.2↓)
+ *     *[58]<a href=/r/1>Result 1</a>
+ *     [59]<a href=/r/2>Result 2</a>
+ *
+ * - `[id]` — interactive element with backend index for targeting
+ * - `*` prefix — newly appeared element (incremental diff)
+ * - `|scroll|` — scrollable container with page counts
+ * - `|shadow|` — Shadow DOM boundary
+ * - `|iframe|` — iframe content
+ * - `|table|` — markdown table rendering
+ */
+export declare function generateSnapshotJs(opts?: DomSnapshotOptions): string;