@zumer/snapdom-plugins 1.1.0 → 1.2.0

package/README.md

@@ -172,23 +172,33 @@ snapdom(el, { plugins: [htmlInCanvas()] });
 
 ### `prompt-export`
 
-Adds a `toPrompt()` export method that returns an LLM-ready package: an annotated screenshot, a structured element map with bounding boxes, and a pre-formatted text description. Useful for vision-language models, browser-agent pipelines, visual QA, and any workflow that pairs a capture with structured metadata.
+Adds a `toPrompt()` export method that returns an LLM-ready package: a structured element map with bounding boxes, a pre-formatted prompt text, and (optionally) an annotated screenshot. Tuned for vision-language models, browser-agent pipelines, visual QA, and any workflow that pairs a capture with structured metadata.
 
 ```js
 import { promptExport } from '@zumer/snapdom-plugins/prompt-export';
 
 const result = await snapdom(el, { plugins: [promptExport()] });
-const { image, elements, dimensions, prompt } = await result.toPrompt();
+// Default: no image, just the structured map + prompt text (cheapest)
+const { elements, prompt, dimensions } = await result.toPrompt();
 ```
 
-The returned object:
+To also include the annotated image (for tasks that truly depend on vision):
+
+```js
+const result = await snapdom(el, {
+  plugins: [promptExport({ include: ['image', 'elements', 'prompt'] })]
+});
+const { image, elements, prompt, dimensions } = await result.toPrompt();
+```
+
+The returned object (fields present only if requested via `include`):
 
 | Field | Type | Description |
 |-------|------|-------------|
-| `image` | `string` | Data URL of the (optionally annotated) screenshot |
-| `elements` | `Array` | One entry per detected element: `{ id, tag, type, text, bbox, attributes }` |
-| `dimensions` | `{width, height}` | Image dimensions after `maxImageWidth` scaling |
+| `elements` | `Array` | One entry per detected element: `{ id, tag, type, name, text, bbox, attributes, state?, styles?, covered? }` |
 | `prompt` | `string` | Pre-formatted text describing interactive + semantic elements |
+| `image` | `string` | Data URL of the (optionally annotated) screenshot — **only when `include` contains `'image'`** |
+| `dimensions` | `{width, height}` | Scaled dimensions (always present) |
 
 `elements` is split into two `type`s:
 - `'interactive'` — buttons, links, inputs, `[role]`/`[tabindex]` targets. These get numbered badges overlaid on the screenshot when `annotate` is on.
@@ -196,9 +206,17 @@ The returned object:
 
 Each `bbox` is in pixel coordinates of the returned image (scaled against `maxImageWidth`).
 
+Each interactive entry also carries:
+- `name` — the computed accessible name (aria-label → labelledby → alt → title → labels[0] → textContent)
+- `state` — runtime state: `{ checked, disabled, focus, open, value, selectedText }` (only keys that apply)
+- `styles` — visually-meaningful computed props filtered to drop defaults
+- `covered: true` when another element is painted on top of the bbox center (an agent won't click through a modal)
+
 ```js
 // Example — feed a vision-capable LLM
-const { image, elements } = await result.toPrompt({ maxImageWidth: 1024 });
+const { image, elements } = await result.toPrompt({
+  include: ['image', 'elements', 'prompt']
+});
 
 // image is a data URL → pass as image input
 // elements is JSON → pass as structured context alongside the image
@@ -207,8 +225,11 @@ const { image, elements } = await result.toPrompt({ maxImageWidth: 1024 });
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `annotate` | `boolean` | `true` | Overlay numbered badges on interactive elements |
-| `imageFormat` | `'png' \| 'jpg' \| 'webp'` | `'png'` | Output image format |
+| `include` | `string[]` | `['elements', 'prompt']` | Fields to return. Add `'image'` for tasks that need vision (chart content, layout QA, canvas). Use `['prompt']` for the cheapest text-only mode. |
+| `annotate` | `boolean` | `true` | Overlay numbered badges on interactive elements (only affects the image when included) |
+| `promptMode` | `'compact' \| 'verbose'` | `'compact'` | Prompt text verbosity. Compact omits coords when badges are on the image. |
+| `includeCoords` | `boolean` | `true` | Include bbox in the prompt text |
+| `imageFormat` | `'png' \| 'jpg' \| 'webp'` | `'png'` | Output image format (only used when `image` is included) |
 | `imageQuality` | `number` | `0.8` | Quality for lossy formats (0–1) |
 | `maxImageWidth` | `number` | `1024` | Max width in px; downscales and rescales bboxes if larger |
 | `interactiveSelector` | `string` | see below | CSS selector for the interactive element set |
@@ -219,7 +240,17 @@ Defaults:
 - **interactive**: `a[href], button, input, select, textarea, [role="button"|"link"|"tab"|"menuitem"|"checkbox"|"radio"], [tabindex]:not([tabindex="-1"]), summary, [contenteditable="true"]`
 - **semantic**: `h1–h6, p, li, img[alt], nav, main, article, section, header, footer, label, td, th, figcaption, blockquote, legend`
 
-Both per-call options (`opts.imageFormat`, etc.) and constructor options are supported; per-call wins.
+Both per-call options (`opts.include`, `opts.imageFormat`, etc.) and constructor options are supported; per-call wins.
+
+The image is the most expensive part of `toPrompt()` to produce (canvas draw + data-URL serialization), so the default skips it. Add `'image'` to `include` when the task actually uses vision:
+
+```js
+// Vision-dependent task (chart content, layout QA, visual diff)
+await result.toPrompt({ include: ['image', 'elements', 'prompt'] });
+
+// Pure structured agent loop (cheapest)
+await result.toPrompt({ include: ['prompt'] });
+```
 
 ---
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zumer/snapdom-plugins",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Official plugins for SnapDOM",
   "type": "module",
   "exports": {

package/prompt-export.js

@@ -19,6 +19,11 @@
  * @param {Object}   [options.labelStyle={}]        - Override styles for annotation badges
  * @param {'compact'|'verbose'} [options.promptMode='compact']  - Prompt text verbosity
  * @param {boolean}  [options.includeCoords=true]   - Include bbox in the prompt text
+ * @param {string[]} [options.include]              - Which fields to return. Default
+ *   ['elements', 'prompt']. For vision-dependent tasks (chart content, layout QA,
+ *   canvas) pass ['image', 'elements', 'prompt'] or add 'image' to the array. For
+ *   text-only agent prompts pass ['prompt'] (cheapest — skips canvas draw entirely).
+ *   Accepted values: 'image', 'elements', 'prompt'.
  * @returns {Object} SnapDOM plugin
  */
 
@@ -58,6 +63,12 @@ function isDefaultStyleValue(prop, value) {
   return false;
 }
 
+// Default omits 'image'. Benchmarking showed the text + JSON map is enough
+// to answer most UI-inspection questions and uses ~14× fewer tokens. Pass
+// `include: ['image', 'elements', 'prompt']` explicitly when the task truly
+// depends on vision (charts, canvas content, layout QA).
+const DEFAULT_INCLUDE = ['elements', 'prompt'];
+
 export function promptExport(options = {}) {
   const {
     annotate = true,
@@ -69,6 +80,7 @@ export function promptExport(options = {}) {
     labelStyle = {},
     promptMode = 'compact',
     includeCoords = true,
+    include = DEFAULT_INCLUDE,
   } = options;
 
   return {
@@ -90,13 +102,17 @@ export function promptExport(options = {}) {
       return {
         prompt: async (ctx, opts = {}) => {
           const meta = ctx.__promptMetadata;
+          const wantSet = new Set(opts.include || include || DEFAULT_INCLUDE);
+          const wantImage = wantSet.has('image');
+          const wantElements = wantSet.has('elements');
+          const wantPrompt = wantSet.has('prompt');
+
           if (!meta || !meta.elements.length) {
-            return {
-              image: ctx.export.url,
-              elements: [],
-              dimensions: { width: 0, height: 0 },
-              prompt: '',
-            };
+            const empty = { dimensions: { width: 0, height: 0 } };
+            if (wantImage) empty.image = ctx.export.url;
+            if (wantElements) empty.elements = [];
+            if (wantPrompt) empty.prompt = '';
+            return empty;
           }
 
           const format = opts.imageFormat || imageFormat;
@@ -105,32 +121,40 @@ export function promptExport(options = {}) {
           const mode = opts.promptMode || promptMode;
           const withCoords = opts.includeCoords !== undefined ? opts.includeCoords : includeCoords;
 
-          const img = new Image();
-          img.src = ctx.export.url;
-          await new Promise((res, rej) => {
-            img.onload = res;
-            img.onerror = rej;
-          });
-
-          const ratio = img.naturalWidth > maxWidth ? maxWidth / img.naturalWidth : 1;
-          const w = Math.round(img.naturalWidth * ratio);
-          const h = Math.round(img.naturalHeight * ratio);
-
-          const canvas = document.createElement('canvas');
-          canvas.width = w;
-          canvas.height = h;
-          canvas.getContext('2d').drawImage(img, 0, 0, w, h);
-
-          const mime =
-            format === 'jpg' || format === 'jpeg' ? 'image/jpeg'
-            : format === 'webp' ? 'image/webp'
-            : 'image/png';
-          const dataURL = canvas.toDataURL(mime, quality);
+          // Only load + rasterize the SVG when the caller actually wants the
+          // image. Skipping saves the img decode + canvas draw + toDataURL —
+          // the most expensive steps of this export.
+          let w, h, dataURL;
+          if (wantImage) {
+            const img = new Image();
+            img.src = ctx.export.url;
+            await new Promise((res, rej) => { img.onload = res; img.onerror = rej; });
+            const ratio = img.naturalWidth > maxWidth ? maxWidth / img.naturalWidth : 1;
+            w = Math.round(img.naturalWidth * ratio);
+            h = Math.round(img.naturalHeight * ratio);
+            const canvas = document.createElement('canvas');
+            canvas.width = w;
+            canvas.height = h;
+            canvas.getContext('2d').drawImage(img, 0, 0, w, h);
+            const mime =
+              format === 'jpg' || format === 'jpeg' ? 'image/jpeg'
+              : format === 'webp' ? 'image/webp'
+              : 'image/png';
+            dataURL = canvas.toDataURL(mime, quality);
+          } else {
+            // No image — scale bboxes to the same target width the image would
+            // have used, so downstream callers can still render the map over
+            // a separately-rendered screenshot at the same scale.
+            const sourceW = meta.dimensions.width || 1;
+            const ratio = sourceW > maxWidth ? maxWidth / sourceW : 1;
+            w = Math.round(sourceW * ratio);
+            h = Math.round(meta.dimensions.height * ratio);
+          }
 
           const sx = w / (meta.dimensions.width || 1);
           const sy = h / (meta.dimensions.height || 1);
 
-          const elements = meta.elements.map((el) => ({
+          const scaledElements = meta.elements.map((el) => ({
             ...el,
             bbox: {
               x: Math.round(el.bbox.x * sx),
@@ -140,19 +164,18 @@ export function promptExport(options = {}) {
             },
           }));
 
-          // In compact + annotate, badges already encode positions on the
-          // image itself, so repeating coords in the prompt text is noise.
           const emitCoords = mode === 'verbose' ? true : (withCoords && !annotate);
-          const prompt = mode === 'verbose'
-            ? formatPromptVerbose(elements, { width: w, height: h }, emitCoords)
-            : formatPromptCompact(elements, { width: w, height: h }, emitCoords);
-
-          return {
-            image: dataURL,
-            elements,
-            dimensions: { width: w, height: h },
-            prompt,
-          };
+          const promptText = (wantPrompt)
+            ? (mode === 'verbose'
+                ? formatPromptVerbose(scaledElements, { width: w, height: h }, emitCoords)
+                : formatPromptCompact(scaledElements, { width: w, height: h }, emitCoords))
+            : null;
+
+          const out = { dimensions: { width: w, height: h } };
+          if (wantImage) out.image = dataURL;
+          if (wantElements) out.elements = scaledElements;
+          if (wantPrompt) out.prompt = promptText;
+          return out;
         },
       };
     },