@zumer/snapdom-plugins 1.0.2 → 1.1.0

package/README.md

@@ -170,6 +170,59 @@ snapdom(el, { plugins: [htmlInCanvas()] });
 
 ---
 
+### `prompt-export`
+
+Adds a `toPrompt()` export method that returns an LLM-ready package: an annotated screenshot, a structured element map with bounding boxes, and a pre-formatted text description. Useful for vision-language models, browser-agent pipelines, visual QA, and any workflow that pairs a capture with structured metadata.
+
+```js
+import { promptExport } from '@zumer/snapdom-plugins/prompt-export';
+
+const result = await snapdom(el, { plugins: [promptExport()] });
+const { image, elements, dimensions, prompt } = await result.toPrompt();
+```
+
+The returned object:
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `image` | `string` | Data URL of the (optionally annotated) screenshot |
+| `elements` | `Array` | One entry per detected element: `{ id, tag, type, text, bbox, attributes }` |
+| `dimensions` | `{width, height}` | Image dimensions after `maxImageWidth` scaling |
+| `prompt` | `string` | Pre-formatted text describing interactive + semantic elements |
+
+`elements` is split into two `type`s:
+- `'interactive'` — buttons, links, inputs, `[role]`/`[tabindex]` targets. These get numbered badges overlaid on the screenshot when `annotate` is on.
+- `'semantic'` — headings, paragraphs, `<nav>`, `<main>`, images with `alt`, table cells, etc. Structural context, not overlaid.
+
+Each `bbox` is in pixel coordinates of the returned image (scaled against `maxImageWidth`).
+
+```js
+// Example — feed a vision-capable LLM
+const { image, elements } = await result.toPrompt({ maxImageWidth: 1024 });
+
+// image is a data URL → pass as image input
+// elements is JSON → pass as structured context alongside the image
+// "Click element [3]" → look up elements[3].bbox for real coordinates
+```
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `annotate` | `boolean` | `true` | Overlay numbered badges on interactive elements |
+| `imageFormat` | `'png' \| 'jpg' \| 'webp'` | `'png'` | Output image format |
+| `imageQuality` | `number` | `0.8` | Quality for lossy formats (0–1) |
+| `maxImageWidth` | `number` | `1024` | Max width in px; downscales and rescales bboxes if larger |
+| `interactiveSelector` | `string` | see below | CSS selector for the interactive element set |
+| `semanticSelector` | `string` | see below | CSS selector for the semantic element set |
+| `labelStyle` | `object` | `{}` | Override styles for the numbered badges (`position`, `color`, `backgroundColor`, etc.) |
+
+Defaults:
+- **interactive**: `a[href], button, input, select, textarea, [role="button"|"link"|"tab"|"menuitem"|"checkbox"|"radio"], [tabindex]:not([tabindex="-1"]), summary, [contenteditable="true"]`
+- **semantic**: `h1–h6, p, li, img[alt], nav, main, article, section, header, footer, label, td, th, figcaption, blockquote, legend`
+
+Both per-call options (`opts.imageFormat`, etc.) and constructor options are supported; per-call wins.
+
+---
+
 ## Plugin registration
 
 **Global** (applies to all captures):

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zumer/snapdom-plugins",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "description": "Official plugins for SnapDOM",
   "type": "module",
   "exports": {

package/prompt-export.js

@@ -1,6 +1,8 @@
 /**
  * promptExport – Official SnapDOM Plugin
- * Produces an LLM-ready package: annotated screenshot + structured element map + prompt text.
+ * Produces an LLM-ready package: annotated screenshot + structured element
+ * map + prompt text. Tuned for vision-capable LLMs reading the image + map
+ * together (Set-of-Mark pattern).
  *
  * Usage:
  *   import { promptExport } from '@zumer/snapdom-plugins/prompt-export';
@@ -9,12 +11,14 @@
  *
  * @param {Object} [options]
  * @param {boolean}  [options.annotate=true]        - Overlay numbered badges on interactive elements
- * @param {string}   [options.imageFormat='png']     - Output image format ('png'|'jpg'|'webp')
- * @param {number}   [options.imageQuality=0.8]      - Quality for lossy formats (0..1)
- * @param {number}   [options.maxImageWidth=1024]    - Max width in px (downscales if larger)
- * @param {string}   [options.interactiveSelector]   - Custom CSS selector for interactive elements
- * @param {string}   [options.semanticSelector]      - Custom CSS selector for semantic elements
- * @param {Object}   [options.labelStyle={}]         - Override styles for annotation badges
+ * @param {string}   [options.imageFormat='png']    - Output image format ('png'|'jpg'|'webp')
+ * @param {number}   [options.imageQuality=0.8]     - Quality for lossy formats (0..1)
+ * @param {number}   [options.maxImageWidth=1024]   - Max width in px (downscales if larger)
+ * @param {string}   [options.interactiveSelector]  - Custom CSS selector for interactive elements
+ * @param {string}   [options.semanticSelector]     - Custom CSS selector for semantic elements
+ * @param {Object}   [options.labelStyle={}]        - Override styles for annotation badges
+ * @param {'compact'|'verbose'} [options.promptMode='compact']  - Prompt text verbosity
+ * @param {boolean}  [options.includeCoords=true]   - Include bbox in the prompt text
  * @returns {Object} SnapDOM plugin
  */
 
@@ -27,6 +31,33 @@ const DEFAULT_SEMANTIC =
   'h1, h2, h3, h4, h5, h6, p, li, img[alt], nav, main, article, section, ' +
   'header, footer, label, td, th, figcaption, blockquote, legend';
 
+const COLLECTED_ATTRS = [
+  'role', 'aria-label', 'aria-expanded', 'aria-checked', 'aria-disabled',
+  'alt', 'href', 'placeholder', 'name', 'type', 'value', 'title', 'disabled',
+];
+
+const VISUAL_FIELDS = [
+  'display', 'visibility', 'opacity',
+  'color', 'backgroundColor',
+  'fontSize', 'fontWeight',
+  'cursor', 'overflow',
+];
+
+// Common computed-style values that carry no information. Keeping the
+// `styles` object small is the difference between a useful LLM input and
+// token bloat on every element.
+const VISUAL_SKIP = new Set(['initial', 'normal', 'visible', 'auto', 'static', '0']);
+function isDefaultStyleValue(prop, value) {
+  if (!value) return true;
+  if (VISUAL_SKIP.has(value)) return true;
+  if (prop === 'cursor' && value === 'none') return true;
+  if (prop === 'color' && value === 'rgb(0, 0, 0)') return true;
+  if (prop === 'backgroundColor' && value === 'rgba(0, 0, 0, 0)') return true;
+  if (prop === 'fontWeight' && (value === '400' || value === 'normal')) return true;
+  if (prop === 'opacity' && value === '1') return true;
+  return false;
+}
+
 export function promptExport(options = {}) {
   const {
     annotate = true,
@@ -36,6 +67,8 @@ export function promptExport(options = {}) {
     interactiveSelector = DEFAULT_INTERACTIVE,
     semanticSelector = DEFAULT_SEMANTIC,
     labelStyle = {},
+    promptMode = 'compact',
+    includeCoords = true,
   } = options;
 
   return {
@@ -43,7 +76,10 @@ export function promptExport(options = {}) {
 
     afterClone(ctx) {
       const meta = extractMetadata(ctx.element, interactiveSelector, semanticSelector);
+      // snapdom spreads a fresh ctx for the export phase from ctx.options,
+      // so write to both so the prompt() call below can read it.
       ctx.__promptMetadata = meta;
+      if (ctx.options) ctx.options.__promptMetadata = meta;
 
       if (annotate) {
         addAnnotations(ctx.clone, meta.elements, labelStyle);
@@ -66,6 +102,8 @@ export function promptExport(options = {}) {
           const format = opts.imageFormat || imageFormat;
           const quality = opts.imageQuality || imageQuality;
           const maxWidth = opts.maxImageWidth || maxImageWidth;
+          const mode = opts.promptMode || promptMode;
+          const withCoords = opts.includeCoords !== undefined ? opts.includeCoords : includeCoords;
 
           const img = new Image();
           img.src = ctx.export.url;
@@ -74,23 +112,19 @@ export function promptExport(options = {}) {
             img.onerror = rej;
           });
 
-          const ratio =
-            img.naturalWidth > maxWidth ? maxWidth / img.naturalWidth : 1;
+          const ratio = img.naturalWidth > maxWidth ? maxWidth / img.naturalWidth : 1;
           const w = Math.round(img.naturalWidth * ratio);
           const h = Math.round(img.naturalHeight * ratio);
 
           const canvas = document.createElement('canvas');
           canvas.width = w;
           canvas.height = h;
-          const c2d = canvas.getContext('2d');
-          c2d.drawImage(img, 0, 0, w, h);
+          canvas.getContext('2d').drawImage(img, 0, 0, w, h);
 
           const mime =
-            format === 'jpg' || format === 'jpeg'
-              ? 'image/jpeg'
-              : format === 'webp'
-                ? 'image/webp'
-                : 'image/png';
+            format === 'jpg' || format === 'jpeg' ? 'image/jpeg'
+            : format === 'webp' ? 'image/webp'
+            : 'image/png';
           const dataURL = canvas.toDataURL(mime, quality);
 
           const sx = w / (meta.dimensions.width || 1);
@@ -106,11 +140,18 @@ export function promptExport(options = {}) {
             },
           }));
 
+          // In compact + annotate, badges already encode positions on the
+          // image itself, so repeating coords in the prompt text is noise.
+          const emitCoords = mode === 'verbose' ? true : (withCoords && !annotate);
+          const prompt = mode === 'verbose'
+            ? formatPromptVerbose(elements, { width: w, height: h }, emitCoords)
+            : formatPromptCompact(elements, { width: w, height: h }, emitCoords);
+
           return {
             image: dataURL,
             elements,
             dimensions: { width: w, height: h },
-            prompt: formatPromptText(elements, { width: w, height: h }),
+            prompt,
           };
         },
       };
@@ -118,6 +159,134 @@ export function promptExport(options = {}) {
   };
 }
 
+/* ── Accessible name ──────────────────────────── */
+
+function truncate(str, max) {
+  if (!str) return '';
+  if (str.length <= max) return str;
+  return str.slice(0, max - 1) + '…';
+}
+
+/**
+ * Compute the element's accessible name following a simplified WAI-ARIA order.
+ * This is what an LLM agent reads first to know what the element IS.
+ */
+function accessibleName(el) {
+  const ariaLabel = el.getAttribute('aria-label');
+  if (ariaLabel && ariaLabel.trim()) return ariaLabel.trim();
+
+  const labelledBy = el.getAttribute('aria-labelledby');
+  if (labelledBy) {
+    const root = el.getRootNode();
+    const getById = (id) =>
+      root && typeof root.getElementById === 'function'
+        ? root.getElementById(id)
+        : document.getElementById(id);
+    const parts = labelledBy.trim().split(/\s+/)
+      .map((id) => {
+        const ref = getById(id);
+        return ref ? (ref.textContent || '').trim() : '';
+      })
+      .filter(Boolean);
+    if (parts.length) return parts.join(' ');
+  }
+
+  if (el.tagName === 'IMG') {
+    const alt = el.getAttribute('alt');
+    if (alt && alt.trim()) return alt.trim();
+  }
+
+  const title = el.getAttribute('title');
+  if (title && title.trim()) return title.trim();
+
+  if (el.labels && el.labels[0]) {
+    const t = (el.labels[0].textContent || '').trim();
+    if (t) return t;
+  }
+
+  const text = (el.textContent || '').trim();
+  return text ? truncate(text, 40) : '';
+}
+
+/* ── DOM state ────────────────────────────────── */
+
+/**
+ * Capture real runtime state (not just ARIA attributes). This is what
+ * separates a useful map from a static screenshot annotation: an agent can
+ * see "checkbox is unchecked / select has value=X / details is closed".
+ */
+function computeState(el) {
+  const state = {};
+  try {
+    if (el.matches(':checked')) state.checked = true;
+    if (el.matches(':disabled')) state.disabled = true;
+    if (el.matches(':focus')) state.focus = true;
+  } catch { /* some selectors may fail on exotic nodes */ }
+
+  const tag = el.tagName;
+  if (tag === 'INPUT') {
+    // Checkbox/radio: `value` defaults to "on" when no explicit value attr
+    // is set — noise. The meaningful signal is `checked`. Skip value here.
+    const type = (el.type || 'text').toLowerCase();
+    if (type !== 'checkbox' && type !== 'radio' && el.value) {
+      state.value = el.value;
+    }
+  } else if (tag === 'TEXTAREA') {
+    if (el.value) state.value = el.value;
+  } else if (tag === 'SELECT') {
+    state.value = el.value;
+    const opt = el.options && el.options[el.selectedIndex];
+    if (opt) state.selectedText = opt.text || '';
+  } else if (tag === 'DETAILS') {
+    state.open = !!el.open;
+  } else if (el.hasAttribute && el.hasAttribute('open')) {
+    state.open = true;
+  }
+
+  return Object.keys(state).length ? state : null;
+}
+
+/* ── Visual styles ────────────────────────────── */
+
+/**
+ * Pull a small set of visually-meaningful computed styles. Reads from the
+ * live element (the clone uses class-based styling so its inline style is
+ * empty and a detached `getComputedStyle(cloneNode)` returns initial values).
+ * The computed style of the original is what snapdom used to build the
+ * capture, so the result matches what the screenshot shows.
+ */
+function computeVisualStyles(el) {
+  let cs;
+  try { cs = getComputedStyle(el); } catch { return null; }
+  if (!cs) return null;
+  const out = {};
+  for (const prop of VISUAL_FIELDS) {
+    const v = cs[prop];
+    if (isDefaultStyleValue(prop, v)) continue;
+    out[prop] = v;
+  }
+  return Object.keys(out).length ? out : null;
+}
+
+/* ── Covered detection ────────────────────────── */
+
+/**
+ * True when another element is painted on top of the center of this one.
+ * An agent that knows a button is covered by a modal won't try to click it.
+ */
+function isCovered(el, rect) {
+  if (!rect.width || !rect.height) return false;
+  const cx = rect.left + rect.width / 2;
+  const cy = rect.top + rect.height / 2;
+  if (cx < 0 || cy < 0) return false;
+  const doc = el.ownerDocument || document;
+  if (!doc.elementFromPoint) return false;
+  const top = doc.elementFromPoint(cx, cy);
+  if (!top || top === el) return false;
+  if (el.contains && el.contains(top)) return false;
+  return true;
+}
+
 /* ── Metadata extraction ──────────────────────── */
 
 function extractMetadata(element, interactiveSelector, semanticSelector) {
@@ -151,11 +320,6 @@ function extractMetadata(element, interactiveSelector, semanticSelector) {
   };
 }
 
-const COLLECTED_ATTRS = [
-  'role', 'aria-label', 'aria-expanded', 'aria-checked', 'aria-disabled',
-  'alt', 'href', 'placeholder', 'name', 'type', 'value', 'title', 'disabled',
-];
-
 function buildEntry(el, rootRect, id, type) {
   const rect = el.getBoundingClientRect();
   const bbox = {
@@ -168,15 +332,58 @@ function buildEntry(el, rootRect, id, type) {
   if (bbox.width <= 0 && bbox.height <= 0) return null;
 
   const tag = el.tagName.toLowerCase();
-  const text = (el.textContent || '').trim().slice(0, 200);
+  const maxText = type === 'interactive' ? 200 : 120;
+  const text = (el.textContent || '').trim().slice(0, maxText);
 
   const attributes = {};
   for (const attr of COLLECTED_ATTRS) {
     const val = el.getAttribute(attr);
-    if (val != null) attributes[attr] = val;
+    if (val == null || val === '' || val === 'false') continue;
+    attributes[attr] = val;
   }
 
-  return { id, tag, type, text, bbox, attributes };
+  // For <img>, replace the (potentially long, cache-busted) src with just the
+  // filename — "logo.svg" is more useful to an LLM than the full URL.
+  if (tag === 'img' && el.src) {
+    try {
+      const base = (el.ownerDocument && el.ownerDocument.location)
+        ? el.ownerDocument.location.href
+        : (typeof location !== 'undefined' ? location.href : undefined);
+      const url = base ? new URL(el.src, base) : new URL(el.src);
+      attributes.src = url.pathname.split('/').pop() || el.src;
+    } catch {
+      attributes.src = el.src;
+    }
+  }
+
+  const entry = {
+    id,
+    tag,
+    type,
+    name: accessibleName(el),
+    text,
+    bbox,
+    attributes,
+  };
+
+  const styles = computeVisualStyles(el);
+  if (styles) entry.styles = styles;
+
+  if (type === 'interactive') {
+    const state = computeState(el);
+    if (state) {
+      // Drop state.value when it just echoes attributes.value — no
+      // divergence between the initial HTML attribute and the current
+      // property, nothing for the LLM to learn from the repeat.
+      if (state.value !== undefined && state.value === attributes.value) {
+        delete state.value;
+      }
+      if (Object.keys(state).length) entry.state = state;
+    }
+    if (isCovered(el, rect)) entry.covered = true;
+  }
+
+  return entry;
 }
 
 /* ── Visual annotations ───────────────────────── */
@@ -202,10 +409,14 @@ function addAnnotations(clone, elements, customStyle) {
     const badge = document.createElement('span');
     badge.textContent = String(el.id);
     badge.setAttribute('data-snap-prompt-label', String(el.id));
+    // Center the badge on the element's bbox, not on its top-left corner:
+    // `translate(-50%, -50%)` offsets from the anchor point.
+    const cx = el.bbox.x + el.bbox.width / 2;
+    const cy = el.bbox.y + el.bbox.height / 2;
     Object.assign(badge.style, {
       position: 'absolute',
-      left: `${el.bbox.x}px`,
-      top: `${el.bbox.y}px`,
+      left: `${cx}px`,
+      top: `${cy}px`,
       transform: 'translate(-50%, -50%)',
       minWidth: '18px',
       height: '18px',
@@ -229,11 +440,56 @@ function addAnnotations(clone, elements, customStyle) {
   clone.appendChild(overlay);
 }
 
-/* ── Prompt text formatter ────────────────────── */
+/* ── Prompt text formatters ───────────────────── */
+
+function stateToStr(state) {
+  if (!state) return '';
+  const flags = [];
+  const pairs = [];
+  for (const k of Object.keys(state)) {
+    const v = state[k];
+    if (v === true) flags.push(k);
+    else pairs.push(`${k}=${JSON.stringify(v)}`);
+  }
+  if (!flags.length && !pairs.length) return '';
+  return ` {${[...flags, ...pairs].join(', ')}}`;
+}
+
+function coordsToStr(bbox) {
+  return ` (${bbox.x},${bbox.y} ${bbox.width}×${bbox.height})`;
+}
+
+function formatPromptCompact(elements, dimensions, withCoords) {
+  const lines = [`Screenshot (${dimensions.width}×${dimensions.height}px).`];
+
+  const interactive = elements.filter((e) => e.type === 'interactive');
+  const semantic = elements.filter((e) => e.type === 'semantic');
 
-function formatPromptText(elements, dimensions) {
+  if (interactive.length) {
+    lines.push('', 'Interactive:');
+    for (const el of interactive) {
+      const name = el.name ? ` "${truncate(el.name, 60)}"` : '';
+      const st = stateToStr(el.state);
+      const cov = el.covered ? ' (covered)' : '';
+      const coords = withCoords ? coordsToStr(el.bbox) : '';
+      lines.push(`  [${el.id}] ${el.tag}${name}${st}${cov}${coords}`);
+    }
+  }
+
+  if (semantic.length) {
+    lines.push('', 'Semantic:');
+    for (const el of semantic) {
+      const name = el.name ? ` "${truncate(el.name, 60)}"` : '';
+      lines.push(`  [${el.id}] ${el.tag}${name}`);
+    }
+  }
+
+  return lines.join('\n');
+}
+
+function formatPromptVerbose(elements, dimensions, withCoords) {
   const lines = [
-    `Screenshot of a web page (${dimensions.width}\u00d7${dimensions.height}px).`,
+    `Screenshot of a web page (${dimensions.width}×${dimensions.height}px).`,
     '',
   ];
 
@@ -243,13 +499,13 @@ function formatPromptText(elements, dimensions) {
   if (interactive.length) {
     lines.push('Interactive elements:');
     for (const el of interactive) {
-      const attrParts = Object.entries(el.attributes).map(
-        ([k, v]) => `${k}="${v}"`
-      );
-      const text = el.text ? ` "${truncate(el.text, 60)}"` : '';
-      const pos = `(${el.bbox.x},${el.bbox.y} ${el.bbox.width}\u00d7${el.bbox.height})`;
+      const name = el.name ? ` "${truncate(el.name, 80)}"` : '';
+      const attrParts = Object.entries(el.attributes).map(([k, v]) => `${k}="${v}"`);
       const attrs = attrParts.length ? ' ' + attrParts.join(' ') : '';
-      lines.push(`  [${el.id}] <${el.tag}>${text} ${pos}${attrs}`);
+      const pos = withCoords ? coordsToStr(el.bbox) : '';
+      const st = stateToStr(el.state);
+      const cov = el.covered ? ' (covered)' : '';
+      lines.push(`  [${el.id}] <${el.tag}>${name}${pos}${attrs}${st}${cov}`);
     }
     lines.push('');
   }
@@ -257,20 +513,15 @@ function formatPromptText(elements, dimensions) {
   if (semantic.length) {
     lines.push('Semantic structure:');
     for (const el of semantic) {
-      const text = el.text ? ` "${truncate(el.text, 80)}"` : '';
+      const name = el.name ? ` "${truncate(el.name, 80)}"` : '';
       const attrParts = [];
       if (el.attributes.alt) attrParts.push(`alt="${el.attributes.alt}"`);
       if (el.attributes.role) attrParts.push(`role="${el.attributes.role}"`);
       const attrs = attrParts.length ? ' ' + attrParts.join(' ') : '';
-      lines.push(`  [${el.id}] <${el.tag}>${text}${attrs}`);
+      lines.push(`  [${el.id}] <${el.tag}>${name}${attrs}`);
     }
     lines.push('');
   }
 
   return lines.join('\n');
 }
-
-function truncate(str, max) {
-  if (str.length <= max) return str;
-  return str.slice(0, max - 1) + '\u2026';
-}