cypress-annotate 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +65 -0
- package/DOCS/pipeline.md +258 -0
- package/LICENSE +21 -0
- package/README.md +360 -0
- package/dist/annotate-image.d.ts +60 -0
- package/dist/annotate-image.d.ts.map +1 -0
- package/dist/annotate-image.js +160 -0
- package/dist/annotate.d.ts +9 -0
- package/dist/annotate.d.ts.map +1 -0
- package/dist/annotate.js +82 -0
- package/dist/browser/collect-elements.js +158 -0
- package/dist/browser/measure-target.js +69 -0
- package/dist/claude-reasoner.d.ts +17 -0
- package/dist/claude-reasoner.d.ts.map +1 -0
- package/dist/claude-reasoner.js +192 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +74 -0
- package/dist/cypress/commands.d.ts +50 -0
- package/dist/cypress/commands.d.ts.map +1 -0
- package/dist/cypress/commands.js +68 -0
- package/dist/cypress/failure-explainer.d.ts +45 -0
- package/dist/cypress/failure-explainer.d.ts.map +1 -0
- package/dist/cypress/failure-explainer.js +123 -0
- package/dist/cypress/failure-hook.d.ts +30 -0
- package/dist/cypress/failure-hook.d.ts.map +1 -0
- package/dist/cypress/failure-hook.js +148 -0
- package/dist/cypress/failure-report.d.ts +41 -0
- package/dist/cypress/failure-report.d.ts.map +1 -0
- package/dist/cypress/failure-report.js +14 -0
- package/dist/cypress/failure-selector.d.ts +45 -0
- package/dist/cypress/failure-selector.d.ts.map +1 -0
- package/dist/cypress/failure-selector.js +107 -0
- package/dist/cypress/measure-dom.d.ts +52 -0
- package/dist/cypress/measure-dom.d.ts.map +1 -0
- package/dist/cypress/measure-dom.js +132 -0
- package/dist/cypress/scroll-into-view.d.ts +15 -0
- package/dist/cypress/scroll-into-view.d.ts.map +1 -0
- package/dist/cypress/scroll-into-view.js +17 -0
- package/dist/cypress/task.d.ts +47 -0
- package/dist/cypress/task.d.ts.map +1 -0
- package/dist/cypress/task.js +69 -0
- package/dist/draw.d.ts +19 -0
- package/dist/draw.d.ts.map +1 -0
- package/dist/draw.js +177 -0
- package/dist/finder.d.ts +68 -0
- package/dist/finder.d.ts.map +1 -0
- package/dist/finder.js +64 -0
- package/dist/index.d.ts +32 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +40 -0
- package/dist/inventory.d.ts +33 -0
- package/dist/inventory.d.ts.map +1 -0
- package/dist/inventory.js +88 -0
- package/dist/measure.d.ts +11 -0
- package/dist/measure.d.ts.map +1 -0
- package/dist/measure.js +145 -0
- package/dist/pipeline.d.ts +60 -0
- package/dist/pipeline.d.ts.map +1 -0
- package/dist/pipeline.js +115 -0
- package/dist/types.d.ts +115 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +1 -0
- package/package.json +113 -0
- package/skills/annotate-bug/SKILL.md +121 -0
- package/skills/cypress-annotate/SKILL.md +154 -0
- package/src/browser/collect-elements.js +158 -0
- package/src/browser/measure-target.js +69 -0
package/README.md
ADDED
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
# cypress-annotate
|
|
2
|
+
|
|
3
|
+
A Cypress plugin that draws a pixel-accurate box on exactly what's wrong in a
|
|
4
|
+
screenshot. Call `cy.annotate()` where you already know what broke, or register
|
|
5
|
+
the failure hook and get a box on every failed test, labelled from the
|
|
6
|
+
assertion's own expected and actual values, with no AI involved.
|
|
7
|
+
|
|
8
|
+
`sharp` is the only thing that installs with it. No browser download, nothing
|
|
9
|
+
else to configure.
|
|
10
|
+
|
|
11
|
+
## Getting started
|
|
12
|
+
|
|
13
|
+
### 1. Install
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
npm install --save-dev cypress-annotate
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
### 2. Register the Node task
|
|
20
|
+
|
|
21
|
+
The measuring happens in the browser, the image compositing happens in Node. This
|
|
22
|
+
is the Node half.
|
|
23
|
+
|
|
24
|
+
```ts
|
|
25
|
+
// cypress.config.ts
|
|
26
|
+
import { defineConfig } from 'cypress';
|
|
27
|
+
import { registerAnnotateTasks } from 'cypress-annotate/cypress/task';
|
|
28
|
+
|
|
29
|
+
export default defineConfig({
|
|
30
|
+
e2e: {
|
|
31
|
+
setupNodeEvents(on) {
|
|
32
|
+
registerAnnotateTasks(on);
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
});
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
### 3. Register the command
|
|
39
|
+
|
|
40
|
+
```ts
|
|
41
|
+
// cypress/support/e2e.ts
|
|
42
|
+
import 'cypress-annotate/cypress/commands';
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
That adds `cy.annotate()` and its TypeScript types. Nothing else to import in
|
|
46
|
+
your specs.
|
|
47
|
+
|
|
48
|
+
### 4. Annotate something
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
it('flags the broken promo button', () => {
|
|
52
|
+
cy.visit('/checkout');
|
|
53
|
+
cy.annotate('#promo-apply', { label: 'Apply button escapes its card' });
|
|
54
|
+
});
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
### 5. Find the image
|
|
58
|
+
|
|
59
|
+
It overwrites the screenshot Cypress just recorded, so the annotated version is
|
|
60
|
+
what lands in `cypress/screenshots/`, in your reports, and in CI artifacts. There
|
|
61
|
+
is no second file to collect and no reporter to configure.
|
|
62
|
+
|
|
63
|
+
That's the whole setup. Everything below is optional.
|
|
64
|
+
|
|
65
|
+
## Using `cy.annotate()`
|
|
66
|
+
|
|
67
|
+
```ts
|
|
68
|
+
cy.annotate(selector, options?)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
`selector` is a CSS selector or an array of them. Several at once, each with its
|
|
72
|
+
own label:
|
|
73
|
+
|
|
74
|
+
```ts
|
|
75
|
+
cy.annotate(['#promo-apply', '#order-ref'], {
|
|
76
|
+
label: ['Escapes its card', 'Overflows'],
|
|
77
|
+
});
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Crop tight to the element with the rest of the page dimmed, which is usually what
|
|
81
|
+
you want to paste into a ticket:
|
|
82
|
+
|
|
83
|
+
```ts
|
|
84
|
+
cy.annotate('#promo-apply', { label: 'Escapes its card', crop: true, cropPadding: 80 });
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Assert on where the box actually landed. The command yields the result:
|
|
88
|
+
|
|
89
|
+
```ts
|
|
90
|
+
cy.annotate('#grand-total-value', { label: 'Wrong total' }).then((result) => {
|
|
91
|
+
expect(result.warnings).to.be.empty;
|
|
92
|
+
expect(result.drawnRects[0].width).to.be.greaterThan(0);
|
|
93
|
+
});
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Options
|
|
97
|
+
|
|
98
|
+
| Option | Default | What it does |
|
|
99
|
+
| --- | --- | --- |
|
|
100
|
+
| `label` | none | Text on the box. One string, or one per selector in order. |
|
|
101
|
+
| `name` | slug of the first selector | Screenshot filename. |
|
|
102
|
+
| `crop` | `false` | Crop to the element and dim everything outside it. |
|
|
103
|
+
| `cropPadding` | pipeline default | CSS px of context to keep around the crop. |
|
|
104
|
+
| `shape` | `box` | Shape drawn around the target. |
|
|
105
|
+
| `style` | none | Colours, stroke width, label colour, dim strength. |
|
|
106
|
+
| `scrollIntoView` | `true` | Scroll the first target into view, but only if it isn't already. |
|
|
107
|
+
| `scrollOffset` | `120` | CSS px left above the element when scrolling, so a fixed header doesn't cover it. |
|
|
108
|
+
| `keepRaw` | `false` | Keep the un-annotated image alongside, for pixel assertions. |
|
|
109
|
+
| `fullPage` | `false` | Capture the whole document. See the warning under [Things that will bite you](#things-that-will-bite-you). |
|
|
110
|
+
|
|
111
|
+
### What it yields
|
|
112
|
+
|
|
113
|
+
| Field | Meaning |
|
|
114
|
+
| --- | --- |
|
|
115
|
+
| `outPath` | Where the annotated image was written. |
|
|
116
|
+
| `rawPath` | The un-annotated image, when `keepRaw` is set. |
|
|
117
|
+
| `width`, `height` | Image dimensions in pixels. |
|
|
118
|
+
| `scale` | The device pixel ratio actually measured off the image. |
|
|
119
|
+
| `drawnRects` | Final box positions, in image pixels. |
|
|
120
|
+
| `warnings` | Anything ambiguous, such as a selector matching more than one element. |
|
|
121
|
+
|
|
122
|
+
## Annotating every failed test
|
|
123
|
+
|
|
124
|
+
For "the wrong text showed" and similar assertion failures, you can skip
|
|
125
|
+
`cy.annotate()` entirely and have every failure annotate itself.
|
|
126
|
+
|
|
127
|
+
### 1. Turn off Cypress's own failure screenshot
|
|
128
|
+
|
|
129
|
+
```ts
|
|
130
|
+
// cypress.config.ts
|
|
131
|
+
export default defineConfig({
|
|
132
|
+
screenshotOnRunFailure: false,
|
|
133
|
+
// ...
|
|
134
|
+
});
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
This is required. Otherwise Cypress's automatic screenshot and this hook's
|
|
138
|
+
screenshot race each other, and it becomes ambiguous which one the DOM
|
|
139
|
+
measurement matches.
|
|
140
|
+
|
|
141
|
+
### 2. Register the hook
|
|
142
|
+
|
|
143
|
+
```ts
|
|
144
|
+
// cypress/support/e2e.ts
|
|
145
|
+
import { registerFailureCapture } from 'cypress-annotate/cypress/failure-hook';
|
|
146
|
+
|
|
147
|
+
registerFailureCapture();
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Optionally, `registerFailureCapture({ reportPath: 'out/cypress/failures.json' })`
|
|
151
|
+
to move the report, or pass `style` to change the box colours.
|
|
152
|
+
|
|
153
|
+
### 3. Run your suite as normal
|
|
154
|
+
|
|
155
|
+
Every failed test now produces an annotated screenshot and one record in
|
|
156
|
+
`out/cypress/failures.json`.
|
|
157
|
+
|
|
158
|
+
### What you get
|
|
159
|
+
|
|
160
|
+
On each failure the hook recovers the selector the failing command was targeting,
|
|
161
|
+
using `cy.state('current')`'s own args where they're available and falling back
|
|
162
|
+
to chai-jquery's `<tag#id>` rendering in the error message. It measures that
|
|
163
|
+
element live while the page is still in its failed state, scrolls it into view if
|
|
164
|
+
needed, and draws a box labelled from the assertion's own values:
|
|
165
|
+
|
|
166
|
+
> Expected "THIS WILL NOT MATCH" but got "£244.99"
|
|
167
|
+
|
|
168
|
+
That label comes entirely from `err.expected` and `err.actual`, with **zero LLM
|
|
169
|
+
calls**. Each record in the report carries the spec and test name, the recovered
|
|
170
|
+
`selector` and where it came from, `expected` and `actual`, the `label`, the
|
|
171
|
+
screenshot and annotated image paths, `drawnRect`, page `metrics`, and warnings.
|
|
172
|
+
|
|
173
|
+
**When no selector can be recovered**, as with an existence check, a
|
|
174
|
+
`cy.contains()` chain, or a generic page-state assertion, there's nothing to box
|
|
175
|
+
deterministically. Those failures still get captured for free: a plain screenshot
|
|
176
|
+
plus a live inventory of candidate elements, because a screenshot on its own
|
|
177
|
+
carries no DOM information and the browser session will be gone by the time
|
|
178
|
+
anyone looks at it. Getting Claude to explain those is a separate, explicit step,
|
|
179
|
+
covered in [DOCS/pipeline.md](DOCS/pipeline.md).
|
|
180
|
+
|
|
181
|
+
### Cypress Cloud
|
|
182
|
+
|
|
183
|
+
**Confirmed working against a real recorded run.** A `cy.screenshot()` call taken
|
|
184
|
+
from an `afterEach` hook, overwritten in place with the annotation before the run
|
|
185
|
+
ends, is picked up by `cypress run --record` and uploaded as a normal
|
|
186
|
+
`Screenshot` cloud artifact, under the same category label and through the same
|
|
187
|
+
pipeline as any other Cypress screenshot. The uploaded image showed the box.
|
|
188
|
+
|
|
189
|
+
That confirmation replaced an earlier design that looked obviously safer:
|
|
190
|
+
overwrite Cypress's own *automatic* on-failure screenshot in place, on the theory
|
|
191
|
+
that mutating the file Cypress already tracks removes all doubt. It doesn't work.
|
|
192
|
+
That automatic screenshot includes the full Runner UI, meaning the command log
|
|
193
|
+
sidebar, the browser toolbar, and the app rendered at some auto-computed zoom
|
|
194
|
+
that is never a 1:1 capture, so there's no reliable coordinate math to draw a box
|
|
195
|
+
on it at all. The `cy.screenshot({capture: 'viewport'})` this hook takes is a
|
|
196
|
+
clean, predictable capture, the same one every alignment test in this repo is
|
|
197
|
+
verified against, and the `--record` result above shows that uploading it costs
|
|
198
|
+
nothing in Cloud compatibility.
|
|
199
|
+
|
|
200
|
+
Nothing in this path calls Claude, spends money, or needs `ANTHROPIC_API_KEY`
|
|
201
|
+
available to the test job. Every CI run captures its data for free.
|
|
202
|
+
|
|
203
|
+
## Using this with an AI coding agent
|
|
204
|
+
|
|
205
|
+
The package ships a skill file at `skills/cypress-annotate/SKILL.md` covering the
|
|
206
|
+
setup, every option, and the traps worth knowing. For Claude Code, copy it into
|
|
207
|
+
your project:
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
cp node_modules/cypress-annotate/skills/cypress-annotate/SKILL.md \
|
|
211
|
+
.claude/skills/cypress-annotate/SKILL.md
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
Agents working inside this repo should read [AGENTS.md](AGENTS.md) instead.
|
|
215
|
+
|
|
216
|
+
## Things that will bite you
|
|
217
|
+
|
|
218
|
+
**A pitfall if you write your own chai assertions against `err.expected` and
|
|
219
|
+
`err.actual`:** they hold chai's own quoted rendering of the value rather than
|
|
220
|
+
the raw value. `.should('have.text', 'X')` puts `'X'` into `err.expected` with
|
|
221
|
+
the quote marks included. An early version of this feature got that wrong, and it
|
|
222
|
+
was fixed by running a real Cypress failure and inspecting the field.
|
|
223
|
+
|
|
224
|
+
**Full-page capture repeats fixed elements.** Cypress builds a full-page
|
|
225
|
+
screenshot by scrolling and stitching, so `position: fixed` and `sticky` elements
|
|
226
|
+
are painted several times in one image. Viewport capture is therefore the default
|
|
227
|
+
and `fullPage` is opt-in.
|
|
228
|
+
|
|
229
|
+
**`cy.scrollIntoView()` always scrolls**, whether or not the element is already
|
|
230
|
+
on screen. Calling it unconditionally shoved a perfectly visible element up under
|
|
231
|
+
the fixed header, where the annotation was correct but the element was hidden
|
|
232
|
+
behind it. The command now measures first and only scrolls when the element is
|
|
233
|
+
genuinely not fully visible, leaving `scrollOffset` px of clearance when it does.
|
|
234
|
+
|
|
235
|
+
**Cypress scales the app when the window is smaller than the viewport**, and the
|
|
236
|
+
screenshot may not be `viewport × devicePixelRatio`. `resolveScale()` measures
|
|
237
|
+
the real ratio off the image and corrects for it.
|
|
238
|
+
|
|
239
|
+
**If Cypress fails to start with "bad option: --no-sandbox"**, something in your
|
|
240
|
+
environment has set `ELECTRON_RUN_AS_NODE=1`, which VS Code's extension host
|
|
241
|
+
does. Cypress's binary is Electron, so it starts as plain Node and rejects
|
|
242
|
+
Chromium flags. `npm run test:cypress` goes through
|
|
243
|
+
[scripts/run-cypress.ts](scripts/run-cypress.ts), which strips that variable.
|
|
244
|
+
|
|
245
|
+
**Worth knowing for any visual assertion in Cypress:** Electron's macOS capture
|
|
246
|
+
applies a colour-profile shift. A fixture painted `#FF00E4` comes back as
|
|
247
|
+
`rgb(234,51,221)`. Exact-colour assertions against your stylesheet will fail, so
|
|
248
|
+
the test suite samples the rendered colour out of the image.
|
|
249
|
+
|
|
250
|
+
## The coordinate rules
|
|
251
|
+
|
|
252
|
+
Four things have to line up, and each one is a separate opportunity to be a few
|
|
253
|
+
pixels off. These are the rules the engine follows.
|
|
254
|
+
|
|
255
|
+
**1. `getBoundingClientRect()` is viewport-relative and post-transform.**
|
|
256
|
+
It already accounts for CSS transforms, internally scrolled ancestors, and
|
|
257
|
+
sub-pixel layout. It does *not* account for page scroll or iframes.
|
|
258
|
+
|
|
259
|
+
**2. Scroll offset applies only to a full-page capture, and only for elements
|
|
260
|
+
that scroll.** A viewport screenshot is in the same space the rect is already in,
|
|
261
|
+
so adding scroll there is the classic off-by-a-scroll bug. A full-page screenshot
|
|
262
|
+
is document space, so `rect + scrollX/scrollY`. A `position: fixed` element has
|
|
263
|
+
no document position at all, which the fixed-element note below covers.
|
|
264
|
+
|
|
265
|
+
**3. Device pixel ratio scales CSS px to image px, but verify it rather than
|
|
266
|
+
trusting it.** `resolveScale()` in [src/annotate.ts](src/annotate.ts) checks the
|
|
267
|
+
produced image against `cssWidth × devicePixelRatio`; if they disagree by more
|
|
268
|
+
than 2px it derives the real scale from the image and records a warning. Stroke
|
|
269
|
+
widths, font sizes and padding are all specified in CSS px and multiplied by that
|
|
270
|
+
same scale, so annotations look identical at dpr 1, 2 and 3.
|
|
271
|
+
|
|
272
|
+
**4. Iframes add their content origin, not their border box.** The offset is
|
|
273
|
+
`iframeRect.left + borderLeftWidth + paddingLeft`. A frame's own internal scroll
|
|
274
|
+
needs no correction, since a rect measured inside the frame is already relative
|
|
275
|
+
to that frame's viewport.
|
|
276
|
+
|
|
277
|
+
### The fixed-element trap
|
|
278
|
+
|
|
279
|
+
Chromium's full-page capture paints `position: fixed` elements **once**, at
|
|
280
|
+
whatever scroll offset the capture started from. So for a fixed element there is
|
|
281
|
+
no correct "document position", and adding scroll to it puts the box thousands of
|
|
282
|
+
pixels away from the thing it should mark.
|
|
283
|
+
|
|
284
|
+
The engine detects fixed ancestry during measurement, skips the scroll correction
|
|
285
|
+
for those elements, scrolls to the top before full-page captures by default
|
|
286
|
+
(`scrollToTop`), and emits a warning so the caller knows why.
|
|
287
|
+
|
|
288
|
+
## How alignment is verified
|
|
289
|
+
|
|
290
|
+
Eyeballing a screenshot does not prove pixel accuracy, so nothing here relies on
|
|
291
|
+
it.
|
|
292
|
+
|
|
293
|
+
Every target in [fixtures/test-page.html](fixtures/test-page.html) is painted a
|
|
294
|
+
flat, unique colour with no radius, border or shadow, so the pixels it paints are
|
|
295
|
+
exactly its border box. [scripts/verify.ts](scripts/verify.ts) scans the
|
|
296
|
+
un-annotated capture for that colour, recovers the element's true pixel
|
|
297
|
+
footprint, and compares it against the rect the engine computed.
|
|
298
|
+
|
|
299
|
+
20 cases pass: dpr 1/2/3, CSS transforms, iframe nesting with border and padding,
|
|
300
|
+
internally scrolled containers, below-the-fold targets under both auto-scroll and
|
|
301
|
+
manual scroll, full-page and cropped captures, fixed elements under scroll, and
|
|
302
|
+
the percentage-region fallback in both bases.
|
|
303
|
+
|
|
304
|
+
Residual drift is ≤1.13px at dpr 3, and it comes from scanner quantisation.
|
|
305
|
+
`#target-top` sits at y=147.375 CSS px, so its top edge falls on device pixel
|
|
306
|
+
294.75 and the partially covered row 294 fails the colour test. Its x is an
|
|
307
|
+
integer and the horizontal delta is exactly 0.00. The same 0.5-device-pixel
|
|
308
|
+
signature shows up on every case, always on the axis with a fractional
|
|
309
|
+
coordinate.
|
|
310
|
+
|
|
311
|
+
## Working on this repo
|
|
312
|
+
|
|
313
|
+
```bash
|
|
314
|
+
npm install
|
|
315
|
+
npm run test:cypress # cypress plugin, 7 cases, real Cypress 15
|
|
316
|
+
npm run test:failure-selector # failed-test selector recovery, 12 cases, no browser
|
|
317
|
+
npm run test:cypress-failures # failed-test capture end-to-end, 4 cases, real Cypress 15
|
|
318
|
+
npm run test:svg-escaping # SVG injection regression suite, 5 cases
|
|
319
|
+
npm run typecheck
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
The alignment suite in the section above drives a headless browser directly, so
|
|
323
|
+
it needs a little more setup. [DOCS/pipeline.md](DOCS/pipeline.md) covers it.
|
|
324
|
+
|
|
325
|
+
### Publishing and packaging
|
|
326
|
+
|
|
327
|
+
**The build runs automatically on install.** This repo ships TypeScript source
|
|
328
|
+
and `dist/` is gitignored, so npm's `prepare` script (`tsc` plus a small
|
|
329
|
+
asset-copy step) runs on install. Confirmed by installing the packed tarball into
|
|
330
|
+
an empty project and importing each entry point, which is also what the
|
|
331
|
+
`package install` CI job does on every run.
|
|
332
|
+
|
|
333
|
+
Every peer dependency is marked optional, so npm installs none of them for you
|
|
334
|
+
and stays quiet about it, as long as you only use entry points that don't need
|
|
335
|
+
them. The three Cypress entry points need nothing beyond `cypress` itself.
|
|
336
|
+
|
|
337
|
+
That separation was worth proving. The first attempt at the split missed a case:
|
|
338
|
+
`ClaudeReasoner` lived in the same file as several utility functions that the
|
|
339
|
+
root barrel re-exports, so importing anything at all from `'cypress-annotate'`,
|
|
340
|
+
even `annotateImage`, threw `ERR_MODULE_NOT_FOUND` for `@anthropic-ai/sdk`
|
|
341
|
+
whether or not you ever touched the reasoner. Reading the code would never have
|
|
342
|
+
shown it, because an ES module evaluates a whole file's top-level code the moment
|
|
343
|
+
anything is imported from it, peer-optional or otherwise. What caught it was
|
|
344
|
+
installing the built package into an isolated scratch project with no peers
|
|
345
|
+
present and importing each entry point for real. CI now does that on every push.
|
|
346
|
+
|
|
347
|
+
## Known limits
|
|
348
|
+
|
|
349
|
+
- Label pill widths are estimated from an average glyph ratio, because librsvg
|
|
350
|
+
gives no text measurement API. Long labels get slightly generous padding.
|
|
351
|
+
- Only Chromium is exercised. The fixed-element behaviour in rule 2 is
|
|
352
|
+
specifically a Chromium capture behaviour and would need re-checking elsewhere.
|
|
353
|
+
|
|
354
|
+
## Also in this package
|
|
355
|
+
|
|
356
|
+
The same coordinate engine powers two things the Cypress plugin never touches: a
|
|
357
|
+
general annotation pipeline you can point at any URL, and a Claude-based
|
|
358
|
+
reasoning layer that decides *what's* wrong before drawing the box. Both are
|
|
359
|
+
optional and neither installs by default.
|
|
360
|
+
[DOCS/pipeline.md](DOCS/pipeline.md) covers them.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { AnnotationStyle, CssRect, PageMetrics, PixelRect, ShapeKind } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Annotating an image that somebody else captured.
|
|
4
|
+
*
|
|
5
|
+
* Nothing here knows about Playwright, CDP, or any particular browser driver -
|
|
6
|
+
* it takes a screenshot, the page metrics that were true when it was taken, and
|
|
7
|
+
* rectangles in CSS pixels. That is deliberate: the screenshot may come from a
|
|
8
|
+
* Playwright MCP server, a Chrome DevTools MCP server, or a Cypress run, and the
|
|
9
|
+
* coordinate maths is identical in every case.
|
|
10
|
+
*/
|
|
11
|
+
export interface MeasuredTarget {
|
|
12
|
+
/** Viewport-relative CSS px, straight from getBoundingClientRect(). */
|
|
13
|
+
viewportRect: CssRect;
|
|
14
|
+
/**
|
|
15
|
+
* Document-relative CSS px. Derived from viewportRect and scroll when
|
|
16
|
+
* omitted, which is correct for everything that is not position:fixed.
|
|
17
|
+
*/
|
|
18
|
+
documentRect?: CssRect;
|
|
19
|
+
/** Fixed elements do not move with scroll, so they get no scroll correction. */
|
|
20
|
+
isFixed?: boolean;
|
|
21
|
+
label?: string;
|
|
22
|
+
shape?: ShapeKind;
|
|
23
|
+
style?: AnnotationStyle;
|
|
24
|
+
/** Used only in warnings, to say which target had a problem. */
|
|
25
|
+
describe?: string;
|
|
26
|
+
}
|
|
27
|
+
export interface AnnotateImageOptions {
|
|
28
|
+
metrics: PageMetrics;
|
|
29
|
+
/** Whether the image holds just the viewport or the whole document. */
|
|
30
|
+
capture?: 'viewport' | 'fullPage';
|
|
31
|
+
/** Crop to the annotated area plus padding. */
|
|
32
|
+
crop?: boolean;
|
|
33
|
+
/** Context to keep around the target when cropping, CSS px. Default 40. */
|
|
34
|
+
cropPadding?: number;
|
|
35
|
+
style?: AnnotationStyle;
|
|
36
|
+
}
|
|
37
|
+
export interface AnnotateImageResult {
|
|
38
|
+
image: Buffer;
|
|
39
|
+
/** The same pixels without the overlay, cropped identically. */
|
|
40
|
+
rawImage: Buffer;
|
|
41
|
+
width: number;
|
|
42
|
+
height: number;
|
|
43
|
+
/** Image pixels per CSS pixel actually used. */
|
|
44
|
+
scale: number;
|
|
45
|
+
drawnRects: PixelRect[];
|
|
46
|
+
warnings: string[];
|
|
47
|
+
}
|
|
48
|
+
export declare function resolveStyle(...layers: (AnnotationStyle | undefined)[]): Required<AnnotationStyle>;
|
|
49
|
+
/**
|
|
50
|
+
* The image-pixels-per-CSS-pixel ratio actually used for this capture.
|
|
51
|
+
*
|
|
52
|
+
* `devicePixelRatio` is the intended answer, but it is only a claim about the
|
|
53
|
+
* browser - not about the file. Agent browser tools routinely downscale
|
|
54
|
+
* screenshots to keep them cheap to send, and a capture that has been resized
|
|
55
|
+
* behind your back is exactly what makes a box drift. So the image is measured
|
|
56
|
+
* and the reported ratio is treated as a hint.
|
|
57
|
+
*/
|
|
58
|
+
export declare function resolveScale(imageWidth: number, cssWidth: number, devicePixelRatio: number, warnings: string[]): number;
|
|
59
|
+
export declare function annotateImage(screenshot: Buffer, targets: MeasuredTarget[], options: AnnotateImageOptions): Promise<AnnotateImageResult>;
|
|
60
|
+
//# sourceMappingURL=annotate-image.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"annotate-image.d.ts","sourceRoot":"","sources":["../src/annotate-image.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,eAAe,EACf,OAAO,EACP,WAAW,EACX,SAAS,EACT,SAAS,EACV,MAAM,YAAY,CAAC;AAEpB;;;;;;;;GAQG;AAEH,MAAM,WAAW,cAAc;IAC7B,uEAAuE;IACvE,YAAY,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gFAAgF;IAChF,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,WAAW,CAAC;IACrB,uEAAuE;IACvE,OAAO,CAAC,EAAE,UAAU,GAAG,UAAU,CAAC;IAClC,+CAA+C;IAC/C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,2EAA2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,gEAAgE;IAChE,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAYD,wBAAgB,YAAY,CAAC,GAAG,MAAM,EAAE,CAAC,eAAe,GAAG,SAAS,CAAC,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,CAElG;AAiCD;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC1B,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,gBAAgB,EAAE,MAAM,EACxB,QAAQ,EAAE,MAAM,EAAE,GACjB,MAAM,CAgBR;AAYD,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,cAAc,EAAE,EACzB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,mBAAmB,CAAC,CAkG9B"}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import sharp from 'sharp';
|
|
2
|
+
import { buildOverlaySvg, compositeOverlay } from './draw.js';
|
|
3
|
+
const DEFAULT_STYLE = {
|
|
4
|
+
color: '#FF3B30',
|
|
5
|
+
strokeWidth: 3,
|
|
6
|
+
padding: 4,
|
|
7
|
+
radius: 6,
|
|
8
|
+
dimOutside: 0,
|
|
9
|
+
labelFontSize: 14,
|
|
10
|
+
labelColor: '#FFFFFF',
|
|
11
|
+
};
|
|
12
|
+
export function resolveStyle(...layers) {
|
|
13
|
+
return Object.assign({}, DEFAULT_STYLE, ...layers.map((l) => l ?? {}));
|
|
14
|
+
}
|
|
15
|
+
/** Convert CSS-pixel style values into image pixels at the capture's scale. */
|
|
16
|
+
function toDeviceStyle(style, scale) {
|
|
17
|
+
return {
|
|
18
|
+
color: style.color,
|
|
19
|
+
strokeWidth: style.strokeWidth * scale,
|
|
20
|
+
radius: style.radius * scale,
|
|
21
|
+
dimOutside: style.dimOutside,
|
|
22
|
+
labelFontSize: style.labelFontSize * scale,
|
|
23
|
+
labelColor: style.labelColor,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
function inflate(rect, by) {
|
|
27
|
+
return { x: rect.x - by, y: rect.y - by, width: rect.width + by * 2, height: rect.height + by * 2 };
|
|
28
|
+
}
|
|
29
|
+
function scaleRect(rect, scale) {
|
|
30
|
+
return { x: rect.x * scale, y: rect.y * scale, width: rect.width * scale, height: rect.height * scale };
|
|
31
|
+
}
|
|
32
|
+
function clampRect(rect, width, height) {
|
|
33
|
+
const x = Math.max(0, Math.min(rect.x, width));
|
|
34
|
+
const y = Math.max(0, Math.min(rect.y, height));
|
|
35
|
+
return {
|
|
36
|
+
x,
|
|
37
|
+
y,
|
|
38
|
+
width: Math.max(0, Math.min(rect.x + rect.width, width) - x),
|
|
39
|
+
height: Math.max(0, Math.min(rect.y + rect.height, height) - y),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The image-pixels-per-CSS-pixel ratio actually used for this capture.
|
|
44
|
+
*
|
|
45
|
+
* `devicePixelRatio` is the intended answer, but it is only a claim about the
|
|
46
|
+
* browser - not about the file. Agent browser tools routinely downscale
|
|
47
|
+
* screenshots to keep them cheap to send, and a capture that has been resized
|
|
48
|
+
* behind your back is exactly what makes a box drift. So the image is measured
|
|
49
|
+
* and the reported ratio is treated as a hint.
|
|
50
|
+
*/
|
|
51
|
+
export function resolveScale(imageWidth, cssWidth, devicePixelRatio, warnings) {
|
|
52
|
+
if (cssWidth <= 0) {
|
|
53
|
+
warnings.push('Page metrics reported a zero-width viewport; falling back to devicePixelRatio.');
|
|
54
|
+
return devicePixelRatio;
|
|
55
|
+
}
|
|
56
|
+
const expected = Math.round(cssWidth * devicePixelRatio);
|
|
57
|
+
if (Math.abs(imageWidth - expected) <= 2)
|
|
58
|
+
return devicePixelRatio;
|
|
59
|
+
const derived = imageWidth / cssWidth;
|
|
60
|
+
warnings.push(`Screenshot is ${imageWidth}px wide but devicePixelRatio ${devicePixelRatio} over ${cssWidth} ` +
|
|
61
|
+
`CSS px predicted ${expected}px - the image was probably rescaled. ` +
|
|
62
|
+
`Using the measured scale ${derived.toFixed(4)} instead.`);
|
|
63
|
+
return derived;
|
|
64
|
+
}
|
|
65
|
+
function documentRectOf(target, metrics) {
|
|
66
|
+
if (target.documentRect)
|
|
67
|
+
return target.documentRect;
|
|
68
|
+
if (target.isFixed)
|
|
69
|
+
return { ...target.viewportRect };
|
|
70
|
+
return {
|
|
71
|
+
...target.viewportRect,
|
|
72
|
+
x: target.viewportRect.x + metrics.scrollX,
|
|
73
|
+
y: target.viewportRect.y + metrics.scrollY,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
export async function annotateImage(screenshot, targets, options) {
|
|
77
|
+
if (targets.length === 0)
|
|
78
|
+
throw new Error('No targets supplied.');
|
|
79
|
+
const { metrics } = options;
|
|
80
|
+
const capture = options.capture ?? 'viewport';
|
|
81
|
+
const fullPage = capture === 'fullPage';
|
|
82
|
+
const warnings = [];
|
|
83
|
+
const meta = await sharp(screenshot).metadata();
|
|
84
|
+
const imageWidth = meta.width ?? 0;
|
|
85
|
+
const imageHeight = meta.height ?? 0;
|
|
86
|
+
if (imageWidth === 0 || imageHeight === 0)
|
|
87
|
+
throw new Error('Could not read the screenshot dimensions.');
|
|
88
|
+
const cssWidth = fullPage ? metrics.documentWidth : metrics.viewportWidth;
|
|
89
|
+
const scale = resolveScale(imageWidth, cssWidth, metrics.devicePixelRatio, warnings);
|
|
90
|
+
for (const target of targets) {
|
|
91
|
+
const name = target.describe ?? 'target';
|
|
92
|
+
if (target.viewportRect.width === 0 || target.viewportRect.height === 0) {
|
|
93
|
+
warnings.push(`${name} has a zero-sized box.`);
|
|
94
|
+
}
|
|
95
|
+
if (target.isFixed && fullPage) {
|
|
96
|
+
warnings.push(`${name} is position:fixed, so it has no stable document position in a full-page capture; ` +
|
|
97
|
+
`annotated where it was painted.`);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const sourceRects = targets.map((target) => {
|
|
101
|
+
const style = resolveStyle(options.style, target.style);
|
|
102
|
+
const base = fullPage ? documentRectOf(target, metrics) : target.viewportRect;
|
|
103
|
+
return scaleRect(inflate(base, style.padding), scale);
|
|
104
|
+
});
|
|
105
|
+
let finalImage = screenshot;
|
|
106
|
+
let finalWidth = imageWidth;
|
|
107
|
+
let finalHeight = imageHeight;
|
|
108
|
+
let originX = 0;
|
|
109
|
+
let originY = 0;
|
|
110
|
+
if (options.crop) {
|
|
111
|
+
const pad = (options.cropPadding ?? 40) * scale;
|
|
112
|
+
const union = sourceRects.reduce((acc, r) => {
|
|
113
|
+
const x = Math.min(acc.x, r.x);
|
|
114
|
+
const y = Math.min(acc.y, r.y);
|
|
115
|
+
return {
|
|
116
|
+
x,
|
|
117
|
+
y,
|
|
118
|
+
width: Math.max(acc.x + acc.width, r.x + r.width) - x,
|
|
119
|
+
height: Math.max(acc.y + acc.height, r.y + r.height) - y,
|
|
120
|
+
};
|
|
121
|
+
}, { ...sourceRects[0] });
|
|
122
|
+
const crop = clampRect(inflate(union, pad), imageWidth, imageHeight);
|
|
123
|
+
const box = {
|
|
124
|
+
left: Math.floor(crop.x),
|
|
125
|
+
top: Math.floor(crop.y),
|
|
126
|
+
width: Math.max(1, Math.ceil(crop.width)),
|
|
127
|
+
height: Math.max(1, Math.ceil(crop.height)),
|
|
128
|
+
};
|
|
129
|
+
if (box.left + box.width > imageWidth)
|
|
130
|
+
box.width = imageWidth - box.left;
|
|
131
|
+
if (box.top + box.height > imageHeight)
|
|
132
|
+
box.height = imageHeight - box.top;
|
|
133
|
+
finalImage = await sharp(screenshot).extract(box).png().toBuffer();
|
|
134
|
+
finalWidth = box.width;
|
|
135
|
+
finalHeight = box.height;
|
|
136
|
+
originX = box.left;
|
|
137
|
+
originY = box.top;
|
|
138
|
+
}
|
|
139
|
+
const drawnRects = sourceRects.map((r) => ({ ...r, x: r.x - originX, y: r.y - originY }));
|
|
140
|
+
const specs = drawnRects.map((rect, i) => {
|
|
141
|
+
const target = targets[i];
|
|
142
|
+
const style = resolveStyle(options.style, target?.style);
|
|
143
|
+
return {
|
|
144
|
+
rect,
|
|
145
|
+
label: target?.label,
|
|
146
|
+
shape: target?.shape ?? 'box',
|
|
147
|
+
style: toDeviceStyle(style, scale),
|
|
148
|
+
};
|
|
149
|
+
});
|
|
150
|
+
const image = await compositeOverlay(finalImage, buildOverlaySvg(finalWidth, finalHeight, specs));
|
|
151
|
+
return {
|
|
152
|
+
image,
|
|
153
|
+
rawImage: finalImage,
|
|
154
|
+
width: finalWidth,
|
|
155
|
+
height: finalHeight,
|
|
156
|
+
scale,
|
|
157
|
+
drawnRects,
|
|
158
|
+
warnings,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Page } from 'playwright';
|
|
2
|
+
import type { Annotation, AnnotateOptions, AnnotateResult } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Playwright-driven annotation: position the page, measure, capture, and hand
|
|
5
|
+
* the pixels to `annotateImage`, which owns all the coordinate maths and is
|
|
6
|
+
* shared with the driver-agnostic path.
|
|
7
|
+
*/
|
|
8
|
+
export declare function annotate(page: Page, annotations: Annotation[], options?: AnnotateOptions): Promise<AnnotateResult>;
|
|
9
|
+
//# sourceMappingURL=annotate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"annotate.d.ts","sourceRoot":"","sources":["../src/annotate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,cAAc,EAAmB,MAAM,YAAY,CAAC;AAE/F;;;;GAIG;AACH,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,UAAU,EAAE,EACzB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC,CA+EzB"}
|
package/dist/annotate.js
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { annotateImage } from './annotate-image.js';
|
|
2
|
+
import { measureElement, measureRegion, readPageMetrics, scrollElementIntoView } from './measure.js';
|
|
3
|
+
/**
|
|
4
|
+
* Playwright-driven annotation: position the page, measure, capture, and hand
|
|
5
|
+
* the pixels to `annotateImage`, which owns all the coordinate maths and is
|
|
6
|
+
* shared with the driver-agnostic path.
|
|
7
|
+
*/
|
|
8
|
+
export async function annotate(page, annotations, options = {}) {
|
|
9
|
+
if (annotations.length === 0)
|
|
10
|
+
throw new Error('No annotations supplied.');
|
|
11
|
+
const mode = options.mode ?? 'viewport';
|
|
12
|
+
const fullPageCapture = mode !== 'viewport';
|
|
13
|
+
const warnings = [];
|
|
14
|
+
// Positioning happens before measurement: any scroll invalidates every rect.
|
|
15
|
+
if (fullPageCapture && (options.scrollToTop ?? true)) {
|
|
16
|
+
await page.evaluate(() => window.scrollTo(0, 0));
|
|
17
|
+
}
|
|
18
|
+
else if (mode === 'viewport' && (options.scrollIntoView ?? true)) {
|
|
19
|
+
// A region is defined relative to wherever the page already is, so
|
|
20
|
+
// scrolling to reveal it would invalidate the very percentages given.
|
|
21
|
+
const first = annotations[0];
|
|
22
|
+
if (first?.selector)
|
|
23
|
+
await scrollElementIntoView(page, first.selector);
|
|
24
|
+
}
|
|
25
|
+
await page.waitForTimeout(60);
|
|
26
|
+
const metrics = await readPageMetrics(page);
|
|
27
|
+
const elements = [];
|
|
28
|
+
for (const annotation of annotations) {
|
|
29
|
+
if (annotation.selector && annotation.region) {
|
|
30
|
+
throw new Error('An annotation takes either a selector or a region, not both.');
|
|
31
|
+
}
|
|
32
|
+
if (annotation.selector) {
|
|
33
|
+
elements.push(await measureElement(page, annotation.selector, metrics));
|
|
34
|
+
}
|
|
35
|
+
else if (annotation.region) {
|
|
36
|
+
elements.push(measureRegion(annotation.region, metrics));
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
throw new Error('An annotation needs either a selector or a region.');
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
for (const element of elements) {
|
|
43
|
+
if (element.matchCount > 1) {
|
|
44
|
+
warnings.push(`Selector "${element.selector}" matched ${element.matchCount} elements; annotated the first.`);
|
|
45
|
+
}
|
|
46
|
+
if (mode === 'viewport' && !element.inViewport) {
|
|
47
|
+
warnings.push(`Element "${element.selector}" is outside the viewport; its box will be clipped.`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const screenshot = await page.screenshot({
|
|
51
|
+
fullPage: fullPageCapture,
|
|
52
|
+
animations: 'disabled',
|
|
53
|
+
caret: 'hide',
|
|
54
|
+
type: 'png',
|
|
55
|
+
});
|
|
56
|
+
const targets = elements.map((element, i) => ({
|
|
57
|
+
viewportRect: element.viewportRect,
|
|
58
|
+
documentRect: element.documentRect,
|
|
59
|
+
isFixed: element.isFixed,
|
|
60
|
+
label: annotations[i]?.label,
|
|
61
|
+
shape: annotations[i]?.shape,
|
|
62
|
+
style: annotations[i]?.style,
|
|
63
|
+
describe: `Element "${element.selector}"`,
|
|
64
|
+
}));
|
|
65
|
+
const result = await annotateImage(screenshot, targets, {
|
|
66
|
+
metrics,
|
|
67
|
+
capture: fullPageCapture ? 'fullPage' : 'viewport',
|
|
68
|
+
crop: mode === 'element',
|
|
69
|
+
cropPadding: options.cropPadding,
|
|
70
|
+
style: options.style,
|
|
71
|
+
});
|
|
72
|
+
return {
|
|
73
|
+
image: result.image,
|
|
74
|
+
rawImage: result.rawImage,
|
|
75
|
+
width: result.width,
|
|
76
|
+
height: result.height,
|
|
77
|
+
metrics,
|
|
78
|
+
elements,
|
|
79
|
+
drawnRects: result.drawnRects,
|
|
80
|
+
warnings: [...warnings, ...result.warnings],
|
|
81
|
+
};
|
|
82
|
+
}
|