@jackwener/opencli 1.1.1 → 1.2.1

package/dist/browser/dom-helpers.js

@@ -10,8 +10,21 @@ export function clickJs(ref) {
     return `
     (() => {
       const ref = ${safeRef};
-      const el = document.querySelector('[data-ref="' + ref + '"]')
-        || document.querySelectorAll('a, button, input, [role="button"], [tabindex]')[parseInt(ref, 10) || 0];
+      // 1. data-opencli-ref (set by snapshot engine)
+      let el = document.querySelector('[data-opencli-ref="' + ref + '"]');
+      // 2. data-ref (legacy)
+      if (!el) el = document.querySelector('[data-ref="' + ref + '"]');
+      // 3. CSS selector
+      if (!el && ref.match(/^[a-zA-Z#.\\[]/)) {
+        try { el = document.querySelector(ref); } catch {}
+      }
+      // 4. Numeric index into interactive elements
+      if (!el) {
+        const idx = parseInt(ref, 10);
+        if (!isNaN(idx)) {
+          el = document.querySelectorAll('a, button, input, select, textarea, [role="button"], [tabindex]:not([tabindex="-1"])')[idx];
+        }
+      }
       if (!el) throw new Error('Element not found: ' + ref);
       el.scrollIntoView({ behavior: 'instant', block: 'center' });
       el.click();
@@ -26,13 +39,31 @@ export function typeTextJs(ref, text) {
     return `
     (() => {
       const ref = ${safeRef};
-      const el = document.querySelector('[data-ref="' + ref + '"]')
-        || document.querySelectorAll('input, textarea, [contenteditable]')[parseInt(ref, 10) || 0];
+      // 1. data-opencli-ref (set by snapshot engine)
+      let el = document.querySelector('[data-opencli-ref="' + ref + '"]');
+      // 2. data-ref (legacy)
+      if (!el) el = document.querySelector('[data-ref="' + ref + '"]');
+      // 3. CSS selector
+      if (!el && ref.match(/^[a-zA-Z#.\\[]/)) {
+        try { el = document.querySelector(ref); } catch {}
+      }
+      // 4. Numeric index into typeable elements
+      if (!el) {
+        const idx = parseInt(ref, 10);
+        if (!isNaN(idx)) {
+          el = document.querySelectorAll('input, textarea, [contenteditable="true"]')[idx];
+        }
+      }
       if (!el) throw new Error('Element not found: ' + ref);
       el.focus();
-      el.value = ${safeText};
-      el.dispatchEvent(new Event('input', { bubbles: true }));
-      el.dispatchEvent(new Event('change', { bubbles: true }));
+      if (el.isContentEditable) {
+        el.textContent = ${safeText};
+        el.dispatchEvent(new Event('input', { bubbles: true }));
+      } else {
+        el.value = ${safeText};
+        el.dispatchEvent(new Event('input', { bubbles: true }));
+        el.dispatchEvent(new Event('change', { bubbles: true }));
+      }
       return 'typed';
     })()
   `;

package/dist/browser/dom-snapshot.d.ts

@@ -0,0 +1,86 @@
+/**
+ * DOM Snapshot Engine — Advanced DOM pruning for LLM consumption.
+ *
+ * Inspired by browser-use's multi-layer pruning pipeline, adapted for opencli's
+ * Chrome Extension + CDP architecture. Runs entirely in-page via Runtime.evaluate.
+ *
+ * Pipeline:
+ *   1. Walk DOM tree, collect visibility + layout + interactivity signals
+ *   2. Prune invisible, zero-area, non-content elements
+ *   3. SVG & decoration collapse
+ *   4. Shadow DOM traversal
+ *   5. Same-origin iframe content extraction
+ *   6. Bounding-box parent-child dedup (link/button wrapping children)
+ *   7. Paint-order occlusion detection (overlay/modal coverage)
+ *   8. Attribute whitelist filtering
+ *   9. Table-aware serialization (markdown tables)
+ *  10. Token-efficient serialization with interactive indices
+ *  11. data-ref annotation for click/type targeting
+ *  12. Hidden interactive element hints (scroll-to-reveal)
+ *  13. Incremental diff (mark new elements with *)
+ *
+ * Additional tools:
+ *   - scrollToRefJs(ref) — scroll to a data-opencli-ref element
+ *   - getFormStateJs()  — extract all form fields as structured JSON
+ */
+export interface SnapshotOptions {
+    /** Extra pixels beyond viewport to include (default 800) */
+    viewportExpand?: number;
+    /** Maximum DOM depth to traverse (default 50) */
+    maxDepth?: number;
+    /** Only emit interactive elements and their landmark ancestors */
+    interactiveOnly?: boolean;
+    /** Maximum text content length per node (default 120) */
+    maxTextLength?: number;
+    /** Include scroll position info on scrollable containers (default true) */
+    includeScrollInfo?: boolean;
+    /** Enable bounding-box parent-child dedup (default true) */
+    bboxDedup?: boolean;
+    /** Traverse Shadow DOM roots (default true) */
+    includeShadowDom?: boolean;
+    /** Extract same-origin iframe content (default true) */
+    includeIframes?: boolean;
+    /** Maximum number of iframes to process (default 5) */
+    maxIframes?: number;
+    /** Enable paint-order occlusion detection (default true) */
+    paintOrderCheck?: boolean;
+    /** Annotate interactive elements with data-opencli-ref (default true) */
+    annotateRefs?: boolean;
+    /** Report hidden interactive elements outside viewport (default true) */
+    reportHidden?: boolean;
+    /** Filter ad/noise elements (default true) */
+    filterAds?: boolean;
+    /** Serialize tables as markdown (default true) */
+    markdownTables?: boolean;
+    /** Previous snapshot hash set (JSON array of hashes) for diff marking (default null) */
+    previousHashes?: string | null;
+}
+/**
+ * Generate JS to scroll to an element identified by data-opencli-ref.
+ * Completes the snapshot→action loop: snapshot identifies `[3]<button>`,
+ * caller can then `scrollToRef('3')` to bring it into view.
+ */
+export declare function scrollToRefJs(ref: string): string;
+/**
+ * Generate JS to extract all form field values from the page.
+ * Returns structured JSON: { forms: [{ id, action, fields: [{ tag, type, name, value, ... }] }] }
+ */
+export declare function getFormStateJs(): string;
+/**
+ * Generate JavaScript code that, when evaluated in a page context via CDP
+ * Runtime.evaluate, returns a pruned DOM snapshot string optimised for LLMs.
+ *
+ * The snapshot output format:
+ *   [42]<button type=submit>Search</button>
+ *   |scroll|<div> (0.5↑ 3.2↓)
+ *     *[58]<a href=/r/1>Result 1</a>
+ *     [59]<a href=/r/2>Result 2</a>
+ *
+ * - `[id]` — interactive element with backend index for targeting
+ * - `*` prefix — newly appeared element (incremental diff)
+ * - `|scroll|` — scrollable container with page counts
+ * - `|shadow|` — Shadow DOM boundary
+ * - `|iframe|` — iframe content
+ * - `|table|` — markdown table rendering
+ */
+export declare function generateSnapshotJs(opts?: SnapshotOptions): string;