@zumer/snapdom-plugins 1.2.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -53
- package/agent-map.js +376 -0
- package/gif-export.js +279 -0
- package/html-export.js +63 -0
- package/html-in-canvas.js +33 -17
- package/index.js +5 -2
- package/package.json +6 -3
- package/video-export.js +138 -0
- package/prompt-export.js +0 -550
package/README.md
CHANGED
|
@@ -170,88 +170,154 @@ snapdom(el, { plugins: [htmlInCanvas()] });
|
|
|
170
170
|
|
|
171
171
|
---
|
|
172
172
|
|
|
173
|
-
### `
|
|
173
|
+
### `agent-map`
|
|
174
174
|
|
|
175
|
-
|
|
175
|
+
Produces a Set-of-Mark package for **visual agents**: an annotated screenshot with numbered badges on interactive elements, plus a compact JSON map from badge index → role / accessible name / bbox / state. One call, fully client-side.
|
|
176
176
|
|
|
177
177
|
```js
|
|
178
|
-
import {
|
|
178
|
+
import { agentMap } from '@zumer/snapdom-plugins/agent-map';
|
|
179
179
|
|
|
180
|
-
const result = await snapdom(el, { plugins: [
|
|
181
|
-
|
|
182
|
-
|
|
180
|
+
const result = await snapdom(el, { plugins: [agentMap()] });
|
|
181
|
+
const { image, map, dimensions } = await result.toAgentMap();
|
|
182
|
+
|
|
183
|
+
// image: data URL of the screenshot with numbered red badges overlaid
|
|
184
|
+
// map: [{ i, n, r, b, s? }, …] — index, name, role, bbox, state
|
|
185
|
+
// Agent says "click element 2" → map[2].b gives [x, y, w, h]
|
|
183
186
|
```
|
|
184
187
|
|
|
185
|
-
|
|
188
|
+
Map entry shape (default `fields: 'minimal'`):
|
|
189
|
+
|
|
190
|
+
| Key | Type | Description |
|
|
191
|
+
|-----|------|-------------|
|
|
192
|
+
| `i` | `number` | Index matching the badge drawn on the image |
|
|
193
|
+
| `n` | `string` | Accessible name (aria-label → labelledby → alt → title → labels → textContent, truncated to 60 chars) |
|
|
194
|
+
| `r` | `string` | ARIA-style role (`button`, `link`, `checkbox`, `radio`, `textbox`, `combobox`, `slider`, `heading`, …) — derived from `role` attribute or implicit role of the element |
|
|
195
|
+
| `b` | `[x, y, w, h]` | Bounding box in pixels, scaled against `maxImageWidth` |
|
|
196
|
+
| `s` | `object?` | State: included only when at least one key is meaningful — `checked`, `disabled`, `focus`, `expanded`, `pressed`, `selected`, `value`, `open`, `selectedText`, `covered` |
|
|
197
|
+
|
|
198
|
+
Example map for a checkout form:
|
|
186
199
|
|
|
187
200
|
```js
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
}
|
|
191
|
-
|
|
201
|
+
[
|
|
202
|
+
{ i: 0, n: 'Email', r: 'textbox', b: [28, 80, 280, 34], s: { value: 'ada@example.com' } },
|
|
203
|
+
{ i: 1, n: 'Send product updates', r: 'checkbox', b: [28, 134, 13, 13], s: { checked: true } },
|
|
204
|
+
{ i: 2, n: 'Apply coupon', r: 'button', b: [28, 176, 114, 38], s: { expanded: false } },
|
|
205
|
+
{ i: 3, n: 'Remove coupon', r: 'button', b: [150, 176, 140, 38], s: { disabled: true } },
|
|
206
|
+
{ i: 4, n: 'Pay $53.90', r: 'button', b: [28, 220, 97, 38] }
|
|
207
|
+
]
|
|
192
208
|
```
|
|
193
209
|
|
|
194
|
-
|
|
210
|
+
#### Options
|
|
211
|
+
|
|
212
|
+
| Option | Type | Default | Description |
|
|
213
|
+
|--------|------|---------|-------------|
|
|
214
|
+
| `image` | `'annotated' \| 'raw' \| false` | `'annotated'` | `'annotated'` overlays numbered badges; `'raw'` skips badges; `false` skips image generation entirely (no canvas draw, no toDataURL — cheapest path). |
|
|
215
|
+
| `fields` | `'minimal' \| 'full'` | `'minimal'` | `'full'` adds `t` (raw text content) and `a` (meaningful attributes) per entry. |
|
|
216
|
+
| `semantic` | `boolean` | `false` | Include non-interactive structural elements (headings, paragraphs, landmarks). Off by default — agents act on interactive. |
|
|
217
|
+
| `maxImageWidth` | `number` | `1024` | Downscale target for the image; bboxes rescale to match. |
|
|
218
|
+
| `imageFormat` | `'png' \| 'jpg' \| 'webp'` | `'png'` | Image format (only used when image is rendered). |
|
|
219
|
+
| `imageQuality` | `number` | `0.8` | Quality for lossy formats. |
|
|
220
|
+
| `interactiveSelector` | `string` | see below | CSS selector for interactive elements. |
|
|
221
|
+
| `semanticSelector` | `string` | see below | CSS selector for semantic elements (used when `semantic: true`). |
|
|
222
|
+
| `labelStyle` | `object` | `{}` | Override badge styles. |
|
|
223
|
+
|
|
224
|
+
Defaults:
|
|
225
|
+
- **interactive**: `a[href], button, input, select, textarea, [role="button"|"link"|"tab"|"menuitem"|"checkbox"|"radio"|"switch"|"slider"|"combobox"|"textbox"], [tabindex]:not([tabindex="-1"]), summary, [contenteditable="true"]`
|
|
226
|
+
- **semantic**: `h1–h6, nav, main, article, section, header, footer, figcaption, blockquote, legend, p`
|
|
227
|
+
|
|
228
|
+
Per-call options override constructor options (e.g. `result.toAgentMap({ image: false })`).
|
|
229
|
+
|
|
230
|
+
#### When to use
|
|
195
231
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
| `image` | `string` | Data URL of the (optionally annotated) screenshot — **only when `include` contains `'image'`** |
|
|
201
|
-
| `dimensions` | `{width, height}` | Scaled dimensions (always present) |
|
|
232
|
+
- Visual agents using Set-of-Mark prompting — one call gives you both the labelled image and the coordinate lookup table.
|
|
233
|
+
- Computer-use / browser-agent harnesses that need click coordinates for a vision model's output.
|
|
234
|
+
- Visual QA with an LLM judge — compare before/after captures with structured element identity.
|
|
235
|
+
- Dataset generation for vision-LLM fine-tuning — (image, map) pairs.
|
|
202
236
|
|
|
203
|
-
|
|
204
|
-
- `'interactive'` — buttons, links, inputs, `[role]`/`[tabindex]` targets. These get numbered badges overlaid on the screenshot when `annotate` is on.
|
|
205
|
-
- `'semantic'` — headings, paragraphs, `<nav>`, `<main>`, images with `alt`, table cells, etc. Structural context, not overlaid.
|
|
237
|
+
Because it runs entirely in the browser, it works in contexts where Playwright / Puppeteer can't: Chrome extensions, SaaS web apps capturing the user's own page, Electron apps capturing their own window.
|
|
206
238
|
|
|
207
|
-
|
|
239
|
+
---
|
|
208
240
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
- `styles` — visually-meaningful computed props filtered to drop defaults
|
|
213
|
-
- `covered: true` when another element is painted on top of the bbox center (an agent won't click through a modal)
|
|
241
|
+
### `html-export`
|
|
242
|
+
|
|
243
|
+
Adds a `toHtml()` export that returns the capture as a self-contained, **re-renderable HTML document** (clone + inlined styles/fonts) instead of pixels. It unwraps the SVG `<foreignObject>` SnapDOM already produced, so the markup and CSS match the capture byte-for-byte — nothing is rasterized.
|
|
214
244
|
|
|
215
245
|
```js
|
|
216
|
-
|
|
217
|
-
const { image, elements } = await result.toPrompt({
|
|
218
|
-
include: ['image', 'elements', 'prompt']
|
|
219
|
-
});
|
|
246
|
+
import { htmlExport } from '@zumer/snapdom-plugins/html-export';
|
|
220
247
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
248
|
+
const result = await snapdom(el, { plugins: [htmlExport()] });
|
|
249
|
+
const html = await result.toHtml(); // full <!DOCTYPE html> string
|
|
250
|
+
|
|
251
|
+
// Or download a .html file:
|
|
252
|
+
await result.toHtml({ download: 'snapshot.html' });
|
|
224
253
|
```
|
|
225
254
|
|
|
226
255
|
| Option | Type | Default | Description |
|
|
227
256
|
|--------|------|---------|-------------|
|
|
228
|
-
| `
|
|
229
|
-
| `
|
|
230
|
-
| `promptMode` | `'compact' \| 'verbose'` | `'compact'` | Prompt text verbosity. Compact omits coords when badges are on the image. |
|
|
231
|
-
| `includeCoords` | `boolean` | `true` | Include bbox in the prompt text |
|
|
232
|
-
| `imageFormat` | `'png' \| 'jpg' \| 'webp'` | `'png'` | Output image format (only used when `image` is included) |
|
|
233
|
-
| `imageQuality` | `number` | `0.8` | Quality for lossy formats (0–1) |
|
|
234
|
-
| `maxImageWidth` | `number` | `1024` | Max width in px; downscales and rescales bboxes if larger |
|
|
235
|
-
| `interactiveSelector` | `string` | see below | CSS selector for the interactive element set |
|
|
236
|
-
| `semanticSelector` | `string` | see below | CSS selector for the semantic element set |
|
|
237
|
-
| `labelStyle` | `object` | `{}` | Override styles for the numbered badges (`position`, `color`, `backgroundColor`, etc.) |
|
|
257
|
+
| `fullDocument` | `boolean` | `true` | Wrap output in `<!DOCTYPE html>…`; if `false`, return just `<style>` + the fragment |
|
|
258
|
+
| `filename` | `string` | `'capture.html'` | Download filename when `opts.download` is `true` |
|
|
238
259
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
260
|
+
Per-call `opts`: `download` (`boolean \| string` — `true` triggers download, a string sets the filename), plus `fullDocument` / `filename` overrides. Returns the HTML `string`.
|
|
261
|
+
|
|
262
|
+
---
|
|
242
263
|
|
|
243
|
-
|
|
264
|
+
### `gif-export`
|
|
244
265
|
|
|
245
|
-
|
|
266
|
+
Adds a `toGif()` export that records an **animated GIF** by re-capturing the live element over time and encoding the frames. The GIF89a encoder (median-cut quantization + LZW) is built in — no dependencies. Returns a `Blob` (`image/gif`).
|
|
246
267
|
|
|
247
268
|
```js
|
|
248
|
-
|
|
249
|
-
await result.toPrompt({ include: ['image', 'elements', 'prompt'] });
|
|
269
|
+
import { gifExport } from '@zumer/snapdom-plugins/gif-export';
|
|
250
270
|
|
|
251
|
-
|
|
252
|
-
await result.
|
|
271
|
+
const result = await snapdom(el, { plugins: [gifExport({ fps: 12, duration: 3000 })] });
|
|
272
|
+
const blob = await result.toGif();
|
|
273
|
+
|
|
274
|
+
// Or download directly:
|
|
275
|
+
await result.toGif({ download: 'animation.gif' });
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
| Option | Type | Default | Description |
|
|
279
|
+
|--------|------|---------|-------------|
|
|
280
|
+
| `fps` | `number` | `10` | Frames per second |
|
|
281
|
+
| `duration` | `number` | `2000` | Total duration in ms (ignored if `frames` is set) |
|
|
282
|
+
| `frames` | `number` | — | Explicit frame count (overrides `duration`) |
|
|
283
|
+
| `maxColors` | `number` | `256` | Palette size per frame (2–256) |
|
|
284
|
+
| `background` | `string` | `'#ffffff'` | Color composited under transparent pixels |
|
|
285
|
+
| `scale` | `number` | `1` | Capture scale |
|
|
286
|
+
| `repeat` | `number` | `0` | Loop count (`0` = forever, `-1` = play once) |
|
|
287
|
+
| `filename` | `string` | `'capture.gif'` | Download filename |
|
|
288
|
+
|
|
289
|
+
Per-call `opts` override any constructor option, plus `download` (`boolean \| string`). Each frame is captured live, so CSS animations / dynamic content are recorded as they play.
|
|
290
|
+
|
|
291
|
+
---
|
|
292
|
+
|
|
293
|
+
### `video-export`
|
|
294
|
+
|
|
295
|
+
Adds a `toMp4()` export that records a **video** by re-capturing the live element over time and encoding the frames with the native `MediaRecorder`. Returns a `Blob` whose type reflects what was actually produced.
|
|
296
|
+
|
|
297
|
+
```js
|
|
298
|
+
import { videoExport } from '@zumer/snapdom-plugins/video-export';
|
|
299
|
+
|
|
300
|
+
const result = await snapdom(el, { plugins: [videoExport({ fps: 30, duration: 4000 })] });
|
|
301
|
+
const blob = await result.toMp4();
|
|
302
|
+
|
|
303
|
+
// Or download directly:
|
|
304
|
+
await result.toMp4({ download: true });
|
|
253
305
|
```
|
|
254
306
|
|
|
307
|
+
> **Codec reality:** `MediaRecorder` output depends on the browser. Safari produces MP4 (H.264); Chromium typically produces WebM (VP8/VP9). When MP4 isn't supported the plugin falls back to WebM, warns in the console, and the downloaded file extension is set to `.mp4` / `.webm` accordingly.
|
|
308
|
+
|
|
309
|
+
| Option | Type | Default | Description |
|
|
310
|
+
|--------|------|---------|-------------|
|
|
311
|
+
| `fps` | `number` | `10` | Frames per second |
|
|
312
|
+
| `duration` | `number` | `2000` | Total duration in ms (ignored if `frames` is set) |
|
|
313
|
+
| `frames` | `number` | — | Explicit frame count (overrides `duration`) |
|
|
314
|
+
| `background` | `string` | `'#ffffff'` | Color composited under transparent pixels |
|
|
315
|
+
| `scale` | `number` | `1` | Capture scale |
|
|
316
|
+
| `bitrate` | `number` | — | `videoBitsPerSecond` passed to `MediaRecorder` |
|
|
317
|
+
| `filename` | `string` | — | Download filename (extension auto-set to `.mp4` / `.webm`) |
|
|
318
|
+
|
|
319
|
+
Requires `MediaRecorder` (unavailable in some headless environments). Per-call `opts` override any constructor option, plus `download` (`boolean \| string`).
|
|
320
|
+
|
|
255
321
|
---
|
|
256
322
|
|
|
257
323
|
## Plugin registration
|
package/agent-map.js
ADDED
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* agentMap – Official SnapDOM Plugin
|
|
3
|
+
*
|
|
4
|
+
* Produces a Set-of-Mark package for visual agents: an annotated screenshot
|
|
5
|
+
* with numbered badges on interactive elements, plus a compact JSON map from
|
|
6
|
+
* badge index → role / name / bbox / state. Designed for one-call capture
|
|
7
|
+
* on the client side — visual agents, computer-use harnesses, dataset
|
|
8
|
+
* generation for vision training, visual QA.
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* import { agentMap } from '@zumer/snapdom-plugins/agent-map';
|
|
12
|
+
* const result = await snapdom(el, { plugins: [agentMap()] });
|
|
13
|
+
* const { image, map, dimensions } = await result.toAgentMap();
|
|
14
|
+
*
|
|
15
|
+
* // model reply: "click element 2" → map[2].b gives [x, y, w, h]
|
|
16
|
+
*
|
|
17
|
+
* @param {Object} [options]
|
|
18
|
+
* @param {'annotated'|'raw'|false} [options.image='annotated'] Image output
|
|
19
|
+
* mode. 'annotated' draws numbered badges on the rendered image, 'raw'
|
|
20
|
+
* returns the image without badges, false skips image generation (cheapest).
|
|
21
|
+
* @param {'minimal'|'full'} [options.fields='minimal'] Per-entry shape.
|
|
22
|
+
* 'minimal' returns {i, n, r, b, s?}. 'full' adds {t (text), a (attrs)}.
|
|
23
|
+
* @param {boolean} [options.semantic=false] Include non-interactive semantic
|
|
24
|
+
* elements (headings, paragraphs, nav, main, landmarks). Off by default —
|
|
25
|
+
* agents typically only act on interactive.
|
|
26
|
+
* @param {number} [options.maxImageWidth=1024] Downscale target for the image.
|
|
27
|
+
* @param {'png'|'jpg'|'webp'} [options.imageFormat='png'] Image format.
|
|
28
|
+
* @param {number} [options.imageQuality=0.8] Quality for lossy formats.
|
|
29
|
+
* @param {string} [options.interactiveSelector] CSS selector (default below).
|
|
30
|
+
* @param {string} [options.semanticSelector] CSS selector (default below).
|
|
31
|
+
* @param {Object} [options.labelStyle={}] Override badge styles.
|
|
32
|
+
* @returns {Object} SnapDOM plugin
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
const DEFAULT_INTERACTIVE =
|
|
36
|
+
'a[href], button, input, select, textarea, ' +
|
|
37
|
+
'[role="button"], [role="link"], [role="tab"], [role="menuitem"], [role="checkbox"], [role="radio"], [role="switch"], [role="slider"], [role="combobox"], [role="textbox"], ' +
|
|
38
|
+
'[tabindex]:not([tabindex="-1"]), summary, [contenteditable="true"]';
|
|
39
|
+
|
|
40
|
+
const DEFAULT_SEMANTIC =
|
|
41
|
+
'h1, h2, h3, h4, h5, h6, nav, main, article, section, header, footer, ' +
|
|
42
|
+
'figcaption, blockquote, legend, p';
|
|
43
|
+
|
|
44
|
+
export function agentMap(options = {}) {
|
|
45
|
+
const {
|
|
46
|
+
image = 'annotated',
|
|
47
|
+
fields = 'minimal',
|
|
48
|
+
semantic = false,
|
|
49
|
+
maxImageWidth = 1024,
|
|
50
|
+
imageFormat = 'png',
|
|
51
|
+
imageQuality = 0.8,
|
|
52
|
+
interactiveSelector = DEFAULT_INTERACTIVE,
|
|
53
|
+
semanticSelector = DEFAULT_SEMANTIC,
|
|
54
|
+
labelStyle = {},
|
|
55
|
+
} = options;
|
|
56
|
+
|
|
57
|
+
return {
|
|
58
|
+
name: 'agent-map',
|
|
59
|
+
|
|
60
|
+
afterClone(ctx) {
|
|
61
|
+
const meta = extractMap(
|
|
62
|
+
ctx.element,
|
|
63
|
+
interactiveSelector,
|
|
64
|
+
semantic ? semanticSelector : null,
|
|
65
|
+
fields
|
|
66
|
+
);
|
|
67
|
+
// snapdom's export ctx is a fresh spread of ctx.options, so we stash on
|
|
68
|
+
// both for the agentMap() call below to find it.
|
|
69
|
+
ctx.__agentMapMeta = meta;
|
|
70
|
+
if (ctx.options) ctx.options.__agentMapMeta = meta;
|
|
71
|
+
|
|
72
|
+
if (image === 'annotated') {
|
|
73
|
+
addAnnotations(ctx.clone, meta.map, labelStyle);
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
|
|
77
|
+
defineExports() {
|
|
78
|
+
return {
|
|
79
|
+
agentMap: async (ctx, opts = {}) => {
|
|
80
|
+
const meta = ctx.__agentMapMeta;
|
|
81
|
+
const wantImage = opts.image !== undefined ? opts.image : image;
|
|
82
|
+
|
|
83
|
+
if (!meta || !meta.map.length) {
|
|
84
|
+
const out = { dimensions: { width: 0, height: 0 }, map: [] };
|
|
85
|
+
if (wantImage) out.image = ctx.export.url;
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const format = opts.imageFormat || imageFormat;
|
|
90
|
+
const quality = opts.imageQuality || imageQuality;
|
|
91
|
+
const maxWidth = opts.maxImageWidth || maxImageWidth;
|
|
92
|
+
|
|
93
|
+
// Scale dimensions — whether we rasterize or not, bboxes get resized
|
|
94
|
+
// to the target output size so callers can overlay them on the image.
|
|
95
|
+
let w, h, dataURL;
|
|
96
|
+
if (wantImage) {
|
|
97
|
+
const img = new Image();
|
|
98
|
+
img.src = ctx.export.url;
|
|
99
|
+
await new Promise((res, rej) => { img.onload = res; img.onerror = rej; });
|
|
100
|
+
const ratio = img.naturalWidth > maxWidth ? maxWidth / img.naturalWidth : 1;
|
|
101
|
+
w = Math.round(img.naturalWidth * ratio);
|
|
102
|
+
h = Math.round(img.naturalHeight * ratio);
|
|
103
|
+
const canvas = document.createElement('canvas');
|
|
104
|
+
canvas.width = w;
|
|
105
|
+
canvas.height = h;
|
|
106
|
+
canvas.getContext('2d').drawImage(img, 0, 0, w, h);
|
|
107
|
+
const mime =
|
|
108
|
+
format === 'jpg' || format === 'jpeg' ? 'image/jpeg'
|
|
109
|
+
: format === 'webp' ? 'image/webp'
|
|
110
|
+
: 'image/png';
|
|
111
|
+
dataURL = canvas.toDataURL(mime, quality);
|
|
112
|
+
} else {
|
|
113
|
+
const sourceW = meta.dimensions.width || 1;
|
|
114
|
+
const ratio = sourceW > maxWidth ? maxWidth / sourceW : 1;
|
|
115
|
+
w = Math.round(sourceW * ratio);
|
|
116
|
+
h = Math.round(meta.dimensions.height * ratio);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const sx = w / (meta.dimensions.width || 1);
|
|
120
|
+
const sy = h / (meta.dimensions.height || 1);
|
|
121
|
+
|
|
122
|
+
const scaledMap = meta.map.map(e => {
|
|
123
|
+
const scaled = { ...e, b: [
|
|
124
|
+
Math.round(e.b[0] * sx),
|
|
125
|
+
Math.round(e.b[1] * sy),
|
|
126
|
+
Math.round(e.b[2] * sx),
|
|
127
|
+
Math.round(e.b[3] * sy),
|
|
128
|
+
] };
|
|
129
|
+
return scaled;
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
const out = { dimensions: { width: w, height: h }, map: scaledMap };
|
|
133
|
+
if (wantImage) out.image = dataURL;
|
|
134
|
+
return out;
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
},
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/* ── Role derivation ────────────────────────────── */
|
|
142
|
+
|
|
143
|
+
function deriveRole(el) {
|
|
144
|
+
const explicit = el.getAttribute('role');
|
|
145
|
+
if (explicit) return explicit;
|
|
146
|
+
const tag = el.tagName.toLowerCase();
|
|
147
|
+
const type = (el.type || '').toLowerCase();
|
|
148
|
+
if (tag === 'button') return 'button';
|
|
149
|
+
if (tag === 'a' && el.hasAttribute('href')) return 'link';
|
|
150
|
+
if (tag === 'input') {
|
|
151
|
+
if (type === 'checkbox') return 'checkbox';
|
|
152
|
+
if (type === 'radio') return 'radio';
|
|
153
|
+
if (type === 'range') return 'slider';
|
|
154
|
+
if (type === 'file') return 'file';
|
|
155
|
+
if (type === 'submit' || type === 'button' || type === 'reset' || type === 'image') return 'button';
|
|
156
|
+
return 'textbox';
|
|
157
|
+
}
|
|
158
|
+
if (tag === 'select') return 'combobox';
|
|
159
|
+
if (tag === 'textarea') return 'textbox';
|
|
160
|
+
if (tag === 'summary') return 'button';
|
|
161
|
+
if (tag === 'details') return 'group';
|
|
162
|
+
if (/^h[1-6]$/.test(tag)) return 'heading';
|
|
163
|
+
if (tag === 'nav') return 'navigation';
|
|
164
|
+
if (tag === 'main') return 'main';
|
|
165
|
+
if (tag === 'header') return 'banner';
|
|
166
|
+
if (tag === 'footer') return 'contentinfo';
|
|
167
|
+
if (tag === 'article') return 'article';
|
|
168
|
+
if (tag === 'section') return 'region';
|
|
169
|
+
if (tag === 'p') return 'paragraph';
|
|
170
|
+
if (tag === 'img') return 'image';
|
|
171
|
+
return tag;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/* ── Accessible name ────────────────────────────── */
|
|
175
|
+
|
|
176
|
+
function accessibleName(el) {
|
|
177
|
+
const ariaLabel = el.getAttribute('aria-label');
|
|
178
|
+
if (ariaLabel && ariaLabel.trim()) return ariaLabel.trim();
|
|
179
|
+
|
|
180
|
+
const labelledBy = el.getAttribute('aria-labelledby');
|
|
181
|
+
if (labelledBy) {
|
|
182
|
+
const root = el.getRootNode();
|
|
183
|
+
const getById = (id) =>
|
|
184
|
+
root && typeof root.getElementById === 'function'
|
|
185
|
+
? root.getElementById(id) : document.getElementById(id);
|
|
186
|
+
const parts = labelledBy.trim().split(/\s+/)
|
|
187
|
+
.map(id => { const r = getById(id); return r ? (r.textContent || '').trim() : ''; })
|
|
188
|
+
.filter(Boolean);
|
|
189
|
+
if (parts.length) return parts.join(' ');
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
if (el.tagName === 'IMG' || (el.tagName === 'INPUT' && (el.type || '').toLowerCase() === 'image')) {
|
|
193
|
+
const alt = el.getAttribute('alt');
|
|
194
|
+
if (alt && alt.trim()) return alt.trim();
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const title = el.getAttribute('title');
|
|
198
|
+
if (title && title.trim()) return title.trim();
|
|
199
|
+
|
|
200
|
+
if (el.labels && el.labels[0]) {
|
|
201
|
+
const t = (el.labels[0].textContent || '').trim();
|
|
202
|
+
if (t) return t;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const text = (el.textContent || '').replace(/\s+/g, ' ').trim();
|
|
206
|
+
if (text) return text.length > 60 ? text.slice(0, 59) + '…' : text;
|
|
207
|
+
return '';
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/* ── State extraction ───────────────────────────── */
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Builds the `s` (state) object. Only meaningful states — never default
|
|
214
|
+
* values that add no signal for an agent. Critically, aria-expanded and
|
|
215
|
+
* aria-pressed are included for BOTH values (true and false) because
|
|
216
|
+
* "pressed: false" on a toggle is meaningful information.
|
|
217
|
+
*/
|
|
218
|
+
function deriveState(el, role, rect) {
|
|
219
|
+
const s = {};
|
|
220
|
+
|
|
221
|
+
try {
|
|
222
|
+
if (el.matches(':checked')) s.checked = true;
|
|
223
|
+
else if (role === 'checkbox' || role === 'radio') {
|
|
224
|
+
// include checked:false for form groups where an agent needs to
|
|
225
|
+
// know "unchecked" is a valid state distinct from "not a checkbox".
|
|
226
|
+
s.checked = false;
|
|
227
|
+
}
|
|
228
|
+
if (el.matches(':disabled')) s.disabled = true;
|
|
229
|
+
if (el.matches(':focus')) s.focus = true;
|
|
230
|
+
} catch { /* exotic nodes */ }
|
|
231
|
+
|
|
232
|
+
const expanded = el.getAttribute('aria-expanded');
|
|
233
|
+
if (expanded === 'true') s.expanded = true;
|
|
234
|
+
else if (expanded === 'false') s.expanded = false;
|
|
235
|
+
|
|
236
|
+
const pressed = el.getAttribute('aria-pressed');
|
|
237
|
+
if (pressed === 'true') s.pressed = true;
|
|
238
|
+
else if (pressed === 'false') s.pressed = false;
|
|
239
|
+
|
|
240
|
+
const selected = el.getAttribute('aria-selected');
|
|
241
|
+
if (selected === 'true') s.selected = true;
|
|
242
|
+
else if (selected === 'false' && (role === 'tab' || role === 'option')) s.selected = false;
|
|
243
|
+
|
|
244
|
+
if (el.tagName === 'INPUT') {
|
|
245
|
+
const type = (el.type || 'text').toLowerCase();
|
|
246
|
+
if (type !== 'checkbox' && type !== 'radio' && type !== 'submit' && type !== 'button' && type !== 'reset' && el.value) {
|
|
247
|
+
s.value = el.value;
|
|
248
|
+
}
|
|
249
|
+
} else if (el.tagName === 'TEXTAREA') {
|
|
250
|
+
if (el.value) s.value = el.value;
|
|
251
|
+
} else if (el.tagName === 'SELECT') {
|
|
252
|
+
s.value = el.value;
|
|
253
|
+
const opt = el.options && el.options[el.selectedIndex];
|
|
254
|
+
if (opt) s.selectedText = opt.text || '';
|
|
255
|
+
} else if (el.tagName === 'DETAILS') {
|
|
256
|
+
s.open = !!el.open;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// Covered — element visually occluded by something else at its center.
|
|
260
|
+
if (rect && rect.width && rect.height) {
|
|
261
|
+
const cx = rect.left + rect.width / 2;
|
|
262
|
+
const cy = rect.top + rect.height / 2;
|
|
263
|
+
if (cx >= 0 && cy >= 0) {
|
|
264
|
+
const doc = el.ownerDocument || document;
|
|
265
|
+
if (doc.elementFromPoint) {
|
|
266
|
+
const top = doc.elementFromPoint(cx, cy);
|
|
267
|
+
if (top && top !== el && !(el.contains && el.contains(top))) {
|
|
268
|
+
s.covered = true;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
return Object.keys(s).length ? s : null;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/* ── Map extraction ─────────────────────────────── */
|
|
278
|
+
|
|
279
|
+
function extractMap(element, interactiveSelector, semanticSelector, fields) {
|
|
280
|
+
const rootRect = element.getBoundingClientRect();
|
|
281
|
+
const map = [];
|
|
282
|
+
let i = 0;
|
|
283
|
+
const tracked = new Set();
|
|
284
|
+
|
|
285
|
+
for (const el of element.querySelectorAll(interactiveSelector)) {
|
|
286
|
+
const entry = buildEntry(el, rootRect, i, fields, 'interactive');
|
|
287
|
+
if (entry) { map.push(entry); tracked.add(el); i++; }
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
if (semanticSelector) {
|
|
291
|
+
for (const el of element.querySelectorAll(semanticSelector)) {
|
|
292
|
+
if (tracked.has(el)) continue;
|
|
293
|
+
const entry = buildEntry(el, rootRect, i, fields, 'semantic');
|
|
294
|
+
if (entry) { map.push(entry); i++; }
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
return {
|
|
299
|
+
map,
|
|
300
|
+
dimensions: { width: rootRect.width, height: rootRect.height },
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function buildEntry(el, rootRect, i, fields, kind) {
|
|
305
|
+
const rect = el.getBoundingClientRect();
|
|
306
|
+
const b = [
|
|
307
|
+
Math.round(rect.left - rootRect.left),
|
|
308
|
+
Math.round(rect.top - rootRect.top),
|
|
309
|
+
Math.round(rect.width),
|
|
310
|
+
Math.round(rect.height),
|
|
311
|
+
];
|
|
312
|
+
if (b[2] <= 0 && b[3] <= 0) return null;
|
|
313
|
+
|
|
314
|
+
const role = deriveRole(el);
|
|
315
|
+
const n = accessibleName(el);
|
|
316
|
+
|
|
317
|
+
const entry = { i, n, r: role, b };
|
|
318
|
+
|
|
319
|
+
if (kind === 'interactive') {
|
|
320
|
+
const s = deriveState(el, role, rect);
|
|
321
|
+
if (s) entry.s = s;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
if (fields === 'full') {
|
|
325
|
+
const t = (el.textContent || '').replace(/\s+/g, ' ').trim();
|
|
326
|
+
if (t && t !== n) entry.t = t.length > 160 ? t.slice(0, 159) + '…' : t;
|
|
327
|
+
const a = {};
|
|
328
|
+
for (const name of ['href', 'type', 'name', 'placeholder', 'alt', 'title', 'role', 'aria-label']) {
|
|
329
|
+
const v = el.getAttribute(name);
|
|
330
|
+
if (v && v !== 'false') a[name] = v;
|
|
331
|
+
}
|
|
332
|
+
if (Object.keys(a).length) entry.a = a;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
return entry;
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/* ── Annotations ────────────────────────────────── */
|
|
339
|
+
|
|
340
|
+
function addAnnotations(clone, entries, customStyle) {
|
|
341
|
+
const interactive = entries.filter(e => !e.isSemanticOnly);
|
|
342
|
+
if (!interactive.length) return;
|
|
343
|
+
|
|
344
|
+
const overlay = document.createElement('div');
|
|
345
|
+
overlay.setAttribute('data-snap-agent-overlay', 'true');
|
|
346
|
+
Object.assign(overlay.style, {
|
|
347
|
+
position: 'absolute',
|
|
348
|
+
top: '0', left: '0', width: '100%', height: '100%',
|
|
349
|
+
pointerEvents: 'none',
|
|
350
|
+
zIndex: '2147483647',
|
|
351
|
+
overflow: 'visible',
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
for (const e of interactive) {
|
|
355
|
+
const badge = document.createElement('span');
|
|
356
|
+
badge.textContent = String(e.i);
|
|
357
|
+
const cx = e.b[0] + e.b[2] / 2;
|
|
358
|
+
const cy = e.b[1] + e.b[3] / 2;
|
|
359
|
+
Object.assign(badge.style, {
|
|
360
|
+
position: 'absolute',
|
|
361
|
+
left: cx + 'px', top: cy + 'px',
|
|
362
|
+
transform: 'translate(-50%, -50%)',
|
|
363
|
+
minWidth: '18px', height: '18px',
|
|
364
|
+
lineHeight: '18px', fontSize: '11px', fontWeight: '700',
|
|
365
|
+
fontFamily: 'system-ui, -apple-system, sans-serif',
|
|
366
|
+
color: '#fff', backgroundColor: 'rgba(220, 38, 38, 0.92)',
|
|
367
|
+
borderRadius: '9px', textAlign: 'center', padding: '0 4px',
|
|
368
|
+
boxSizing: 'border-box', boxShadow: '0 1px 3px rgba(0,0,0,0.3)',
|
|
369
|
+
...customStyle,
|
|
370
|
+
});
|
|
371
|
+
overlay.appendChild(badge);
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
clone.style.position = 'relative';
|
|
375
|
+
clone.appendChild(overlay);
|
|
376
|
+
}
|