cypress-annotate 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +40 -28
- package/CHANGELOG.md +142 -0
- package/DOCS/images/crop.png +0 -0
- package/DOCS/images/failure-hook.png +0 -0
- package/DOCS/internals.md +151 -0
- package/README.md +104 -352
- package/dist/annotate-image.d.ts +5 -0
- package/dist/annotate-image.d.ts.map +1 -1
- package/dist/annotate-image.js +16 -2
- package/dist/cypress/commands.d.ts +6 -1
- package/dist/cypress/commands.d.ts.map +1 -1
- package/dist/cypress/commands.js +13 -6
- package/dist/cypress/config.d.ts +42 -0
- package/dist/cypress/config.d.ts.map +1 -0
- package/dist/cypress/config.js +102 -0
- package/dist/cypress/failure-hook.d.ts.map +1 -1
- package/dist/cypress/failure-hook.js +10 -3
- package/dist/cypress/failure-report.d.ts +6 -0
- package/dist/cypress/failure-report.d.ts.map +1 -1
- package/dist/cypress/failure-report.js +14 -4
- package/dist/cypress/task.d.ts +36 -2
- package/dist/cypress/task.d.ts.map +1 -1
- package/dist/cypress/task.js +60 -5
- package/dist/draw.d.ts +4 -0
- package/dist/draw.d.ts.map +1 -1
- package/dist/draw.js +7 -4
- package/dist/index.d.ts +9 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -57
- package/dist/types.d.ts +6 -72
- package/dist/types.d.ts.map +1 -1
- package/package.json +10 -42
- package/skills/cypress-annotate/SKILL.md +51 -9
- package/DOCS/pipeline.md +0 -308
- package/dist/annotate.d.ts +0 -9
- package/dist/annotate.d.ts.map +0 -1
- package/dist/annotate.js +0 -82
- package/dist/browser/collect-elements.js +0 -158
- package/dist/browser/measure-target.js +0 -69
- package/dist/claude-reasoner.d.ts +0 -17
- package/dist/claude-reasoner.d.ts.map +0 -1
- package/dist/claude-reasoner.js +0 -192
- package/dist/cli.d.ts +0 -3
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -84
- package/dist/cypress/failure-explainer.d.ts +0 -45
- package/dist/cypress/failure-explainer.d.ts.map +0 -1
- package/dist/cypress/failure-explainer.js +0 -123
- package/dist/finder.d.ts +0 -68
- package/dist/finder.d.ts.map +0 -1
- package/dist/finder.js +0 -64
- package/dist/inventory.d.ts +0 -33
- package/dist/inventory.d.ts.map +0 -1
- package/dist/inventory.js +0 -88
- package/dist/measure.d.ts +0 -11
- package/dist/measure.d.ts.map +0 -1
- package/dist/measure.js +0 -145
- package/dist/pipeline.d.ts +0 -60
- package/dist/pipeline.d.ts.map +0 -1
- package/dist/pipeline.js +0 -115
- package/skills/annotate-bug/SKILL.md +0 -121
- package/src/browser/collect-elements.js +0 -158
- package/src/browser/measure-target.js +0 -69
package/dist/types.d.ts
CHANGED
|
@@ -25,37 +25,6 @@ export interface PageMetrics {
|
|
|
25
25
|
documentWidth: number;
|
|
26
26
|
documentHeight: number;
|
|
27
27
|
}
|
|
28
|
-
/**
|
|
29
|
-
* Fallback target for when the agent cannot name a clean selector: a rectangle
|
|
30
|
-
* given as percentages, so it is independent of viewport size.
|
|
31
|
-
*/
|
|
32
|
-
export interface Region {
|
|
33
|
-
xPct: number;
|
|
34
|
-
yPct: number;
|
|
35
|
-
widthPct: number;
|
|
36
|
-
heightPct: number;
|
|
37
|
-
/** Percentages of the viewport (default) or of the whole document. */
|
|
38
|
-
basis?: 'viewport' | 'document';
|
|
39
|
-
}
|
|
40
|
-
export interface MeasuredElement {
|
|
41
|
-
/** Human-readable description of what was targeted. */
|
|
42
|
-
selector: string;
|
|
43
|
-
/** Whether this came from a selector match or a percentage region. */
|
|
44
|
-
source: 'selector' | 'region';
|
|
45
|
-
/** Viewport-relative, CSS px, including any ancestor-iframe offsets. */
|
|
46
|
-
viewportRect: CssRect;
|
|
47
|
-
/** Document-relative, CSS px (viewportRect + scroll, except for fixed elements). */
|
|
48
|
-
documentRect: CssRect;
|
|
49
|
-
/** True if the element or an ancestor is position:fixed, so it does not scroll. */
|
|
50
|
-
isFixed: boolean;
|
|
51
|
-
/** True if any part of the element is inside the current viewport. */
|
|
52
|
-
inViewport: boolean;
|
|
53
|
-
/** Extra frames the element was found inside, outermost first. */
|
|
54
|
-
framePath: string[];
|
|
55
|
-
tagName: string;
|
|
56
|
-
/** How many elements the selector matched. Anything above 1 is ambiguous. */
|
|
57
|
-
matchCount: number;
|
|
58
|
-
}
|
|
59
28
|
export type ShapeKind = 'box' | 'circle' | 'arrow';
|
|
60
29
|
export interface AnnotationStyle {
|
|
61
30
|
/** CSS colour for the outline and label background. */
|
|
@@ -70,46 +39,11 @@ export interface AnnotationStyle {
|
|
|
70
39
|
dimOutside?: number;
|
|
71
40
|
labelFontSize?: number;
|
|
72
41
|
labelColor?: string;
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
/**
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
selector?: string | string[];
|
|
80
|
-
/** Percentage-based fallback when no selector is available. */
|
|
81
|
-
region?: Region;
|
|
82
|
-
label?: string;
|
|
83
|
-
shape?: ShapeKind;
|
|
84
|
-
style?: AnnotationStyle;
|
|
85
|
-
}
|
|
86
|
-
export type ScreenshotMode = 'viewport' | 'fullPage' | 'element';
|
|
87
|
-
export interface AnnotateOptions {
|
|
88
|
-
/** Where to screenshot: the viewport, the whole document, or a crop around the element. */
|
|
89
|
-
mode?: ScreenshotMode;
|
|
90
|
-
/** For mode 'element', how much context to keep around the element, in CSS px. */
|
|
91
|
-
cropPadding?: number;
|
|
92
|
-
/**
|
|
93
|
-
* Scroll to the top before capturing a full-page screenshot. Chromium renders
|
|
94
|
-
* fixed elements once, at whatever scroll offset the capture starts from, so
|
|
95
|
-
* starting at the top is the only offset where fixed and static elements agree.
|
|
96
|
-
*/
|
|
97
|
-
scrollToTop?: boolean;
|
|
98
|
-
/** Bring the first annotated element into view before a viewport capture. */
|
|
99
|
-
scrollIntoView?: boolean;
|
|
100
|
-
style?: AnnotationStyle;
|
|
101
|
-
}
|
|
102
|
-
export interface AnnotateResult {
|
|
103
|
-
image: Buffer;
|
|
104
|
-
/** The same capture without the overlay, cropped identically. For verification. */
|
|
105
|
-
rawImage: Buffer;
|
|
106
|
-
/** Pixel dimensions of the returned image. */
|
|
107
|
-
width: number;
|
|
108
|
-
height: number;
|
|
109
|
-
metrics: PageMetrics;
|
|
110
|
-
elements: MeasuredElement[];
|
|
111
|
-
/** Final drawn rectangles, in image pixels. Useful for asserting alignment. */
|
|
112
|
-
drawnRects: PixelRect[];
|
|
113
|
-
warnings: string[];
|
|
42
|
+
/** Label pill background. Defaults to `color`, so one value themes both. */
|
|
43
|
+
labelBackground?: string;
|
|
44
|
+
/** CSS font-family for the label. */
|
|
45
|
+
labelFontFamily?: string;
|
|
46
|
+
/** CSS font-weight for the label. */
|
|
47
|
+
labelFontWeight?: string | number;
|
|
114
48
|
}
|
|
115
49
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,qEAAqE;AACrE,MAAM,WAAW,OAAO;IACtB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,iFAAiF;AACjF,MAAM,WAAW,SAAS;IACxB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,8DAA8D;AAC9D,MAAM,WAAW,WAAW;IAC1B,8DAA8D;IAC9D,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,qEAAqE;AACrE,MAAM,WAAW,OAAO;IACtB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,iFAAiF;AACjF,MAAM,WAAW,SAAS;IACxB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,8DAA8D;AAC9D,MAAM,WAAW,WAAW;IAC1B,8DAA8D;IAC9D,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEnD,MAAM,WAAW,eAAe;IAC9B,uDAAuD;IACvD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4FAA4F;IAC5F,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+BAA+B;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mEAAmE;IACnE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qCAAqC;IACrC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qCAAqC;IACrC,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC"}
|
package/package.json
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cypress-annotate",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Draw a box on exactly what failed. Every failed Cypress test gets an annotated screenshot, in CI, with no AI and no browser download.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cypress",
|
|
7
7
|
"cypress-plugin",
|
|
8
8
|
"cypress-screenshot",
|
|
9
9
|
"screenshot",
|
|
10
|
+
"screenshot-annotation",
|
|
10
11
|
"annotate",
|
|
11
12
|
"annotation",
|
|
12
13
|
"test-failure",
|
|
@@ -16,15 +17,12 @@
|
|
|
16
17
|
"e2e-testing",
|
|
17
18
|
"visual-testing",
|
|
18
19
|
"test-reporting",
|
|
20
|
+
"test-artifacts",
|
|
19
21
|
"ci",
|
|
22
|
+
"github-actions",
|
|
20
23
|
"cypress-cloud",
|
|
21
24
|
"bounding-box",
|
|
22
|
-
"pixel-accurate"
|
|
23
|
-
"playwright",
|
|
24
|
-
"ai",
|
|
25
|
-
"llm",
|
|
26
|
-
"claude",
|
|
27
|
-
"agent"
|
|
25
|
+
"pixel-accurate"
|
|
28
26
|
],
|
|
29
27
|
"license": "MIT",
|
|
30
28
|
"author": "bidemiajala",
|
|
@@ -42,9 +40,6 @@
|
|
|
42
40
|
"type": "module",
|
|
43
41
|
"main": "dist/index.js",
|
|
44
42
|
"types": "dist/index.d.ts",
|
|
45
|
-
"bin": {
|
|
46
|
-
"cypress-annotate": "./dist/cli.js"
|
|
47
|
-
},
|
|
48
43
|
"exports": {
|
|
49
44
|
".": {
|
|
50
45
|
"types": "./dist/index.d.ts",
|
|
@@ -52,12 +47,6 @@
|
|
|
52
47
|
"require": "./dist/index.js",
|
|
53
48
|
"default": "./dist/index.js"
|
|
54
49
|
},
|
|
55
|
-
"./reasoner": {
|
|
56
|
-
"types": "./dist/claude-reasoner.d.ts",
|
|
57
|
-
"import": "./dist/claude-reasoner.js",
|
|
58
|
-
"require": "./dist/claude-reasoner.js",
|
|
59
|
-
"default": "./dist/claude-reasoner.js"
|
|
60
|
-
},
|
|
61
50
|
"./cypress/task": {
|
|
62
51
|
"types": "./dist/cypress/task.d.ts",
|
|
63
52
|
"import": "./dist/cypress/task.js",
|
|
@@ -80,9 +69,6 @@
|
|
|
80
69
|
},
|
|
81
70
|
"typesVersions": {
|
|
82
71
|
"*": {
|
|
83
|
-
"reasoner": [
|
|
84
|
-
"./dist/claude-reasoner.d.ts"
|
|
85
|
-
],
|
|
86
72
|
"cypress/task": [
|
|
87
73
|
"./dist/cypress/task.d.ts"
|
|
88
74
|
],
|
|
@@ -96,55 +82,37 @@
|
|
|
96
82
|
},
|
|
97
83
|
"files": [
|
|
98
84
|
"dist",
|
|
99
|
-
"src/browser",
|
|
100
85
|
"skills",
|
|
101
86
|
"DOCS",
|
|
102
87
|
"AGENTS.md",
|
|
88
|
+
"CHANGELOG.md",
|
|
103
89
|
"README.md",
|
|
104
90
|
"LICENSE"
|
|
105
91
|
],
|
|
106
92
|
"scripts": {
|
|
107
|
-
"build": "tsc -p tsconfig.build.json
|
|
93
|
+
"build": "tsc -p tsconfig.build.json",
|
|
108
94
|
"prepare": "npm run build",
|
|
109
|
-
"annotate": "tsx src/cli.ts",
|
|
110
|
-
"verify": "tsx scripts/verify.ts",
|
|
111
|
-
"verify:image": "tsx scripts/verify-image.ts",
|
|
112
|
-
"annotate-shot": "tsx scripts/annotate-shot.ts",
|
|
113
|
-
"test:reasoner": "tsx scripts/test-reasoner.ts",
|
|
114
95
|
"test:failure-selector": "tsx scripts/test-failure-selector.ts",
|
|
115
96
|
"test:svg-escaping": "tsx scripts/test-svg-escaping.ts",
|
|
116
|
-
"test:cypress": "tsx scripts/run-cypress.ts run --e2e --spec cypress/e2e/annotate.cy.ts",
|
|
97
|
+
"test:cypress": "tsx scripts/run-cypress.ts run --e2e --spec 'cypress/e2e/annotate.cy.ts,cypress/e2e/theme.cy.ts'",
|
|
117
98
|
"test:cypress-failures": "tsx scripts/test-cypress-failures.ts",
|
|
118
|
-
"explain-failures": "tsx scripts/explain-failures.ts",
|
|
119
99
|
"cypress:open": "tsx scripts/run-cypress.ts open",
|
|
120
|
-
"find-bugs": "tsx scripts/find-bugs.ts",
|
|
121
|
-
"demo": "tsx scripts/find-bugs.ts --url fixtures/buggy-checkout.html --replay fixtures/buggy-checkout.expected.json --out-dir out/bugs",
|
|
122
100
|
"typecheck": "tsc --noEmit"
|
|
123
101
|
},
|
|
124
102
|
"dependencies": {
|
|
125
103
|
"sharp": "^0.35.4"
|
|
126
104
|
},
|
|
127
105
|
"peerDependencies": {
|
|
128
|
-
"
|
|
129
|
-
"cypress": ">=13",
|
|
130
|
-
"playwright": "^1.56.1"
|
|
106
|
+
"cypress": ">=13"
|
|
131
107
|
},
|
|
132
108
|
"peerDependenciesMeta": {
|
|
133
|
-
"@anthropic-ai/sdk": {
|
|
134
|
-
"optional": true
|
|
135
|
-
},
|
|
136
109
|
"cypress": {
|
|
137
110
|
"optional": true
|
|
138
|
-
},
|
|
139
|
-
"playwright": {
|
|
140
|
-
"optional": true
|
|
141
111
|
}
|
|
142
112
|
},
|
|
143
113
|
"devDependencies": {
|
|
144
|
-
"@anthropic-ai/sdk": "^0.120.0",
|
|
145
114
|
"@types/node": "^24.10.1",
|
|
146
115
|
"cypress": "^15.19.0",
|
|
147
|
-
"playwright": "^1.56.1",
|
|
148
116
|
"tsx": "^4.20.7",
|
|
149
117
|
"typescript": "^5.9.3"
|
|
150
118
|
}
|
|
@@ -23,8 +23,8 @@ import { registerAnnotateTasks } from 'cypress-annotate/cypress/task';
|
|
|
23
23
|
|
|
24
24
|
export default defineConfig({
|
|
25
25
|
e2e: {
|
|
26
|
-
setupNodeEvents(on) {
|
|
27
|
-
registerAnnotateTasks(on);
|
|
26
|
+
setupNodeEvents(on, config) {
|
|
27
|
+
registerAnnotateTasks(on, config);
|
|
28
28
|
},
|
|
29
29
|
},
|
|
30
30
|
});
|
|
@@ -64,10 +64,17 @@ The offset used is the frame's content origin, so frames with their own border
|
|
|
64
64
|
and padding land correctly and a frame's internal scroll needs no correction.
|
|
65
65
|
Cross-origin frames cannot be measured and fail loudly.
|
|
66
66
|
|
|
67
|
-
Crop tight
|
|
67
|
+
Crop tight, which is what to use for a ticket attachment. Dimming the
|
|
68
|
+
surroundings is a separate `style` setting, since `crop` alone leaves them at
|
|
69
|
+
full brightness:
|
|
68
70
|
|
|
69
71
|
```ts
|
|
70
|
-
cy.annotate('#promo-apply', {
|
|
72
|
+
cy.annotate('#promo-apply', {
|
|
73
|
+
label: 'Escapes its card',
|
|
74
|
+
crop: true,
|
|
75
|
+
cropPadding: 220,
|
|
76
|
+
style: { dimOutside: 0.45 },
|
|
77
|
+
});
|
|
71
78
|
```
|
|
72
79
|
|
|
73
80
|
It overwrites the screenshot Cypress just recorded, so the annotated image is
|
|
@@ -88,7 +95,7 @@ cy.annotate('#grand-total-value', { label: 'Wrong total' }).then((result) => {
|
|
|
88
95
|
| --- | --- | --- |
|
|
89
96
|
| `label` | none | Box text. One string, or one per selector in order. |
|
|
90
97
|
| `name` | slug of first selector | Screenshot filename. |
|
|
91
|
-
| `crop` | `false` | Crop to the element,
|
|
98
|
+
| `crop` | `false` | Crop to the element, keeping `cropPadding` of context. |
|
|
92
99
|
| `cropPadding` | pipeline default | CSS px of context kept around the crop. |
|
|
93
100
|
| `shape` | `box` | Shape drawn around the target. |
|
|
94
101
|
| `style` | none | Colours, stroke width, label colour, dim strength. |
|
|
@@ -157,9 +164,44 @@ colour from the image rather than trusting the stylesheet.
|
|
|
157
164
|
`ELECTRON_RUN_AS_NODE=1`, which VS Code's extension host does. Unset it before
|
|
158
165
|
running Cypress.
|
|
159
166
|
|
|
167
|
+
## Project-wide defaults
|
|
168
|
+
|
|
169
|
+
Colours and the other options are settable once, in cypress.config, so a team
|
|
170
|
+
does not repeat them at every call:
|
|
171
|
+
|
|
172
|
+
```ts
|
|
173
|
+
export default defineConfig({
|
|
174
|
+
env: {
|
|
175
|
+
annotate: {
|
|
176
|
+
style: { color: '#7C3AED', strokeWidth: 4, labelBackground: '#1F1F23' },
|
|
177
|
+
keepRaw: true,
|
|
178
|
+
},
|
|
179
|
+
},
|
|
180
|
+
});
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Cypress serialises `env` into the browser, so the block reaches `cy.annotate()`
|
|
184
|
+
and the failure hook alike, and `CYPRESS_annotate='{"style":{...}}'` overrides it
|
|
185
|
+
for one CI job. Layers merge lowest first: built-in defaults, this block, the
|
|
186
|
+
options passed to a single call, then a per-target style. A misspelled key warns
|
|
187
|
+
rather than being silently ignored.
|
|
188
|
+
|
|
189
|
+
Style fields: `color`, `strokeWidth`, `padding`, `radius`, `dimOutside`,
|
|
190
|
+
`labelFontSize`, `labelColor`, `labelBackground`, `labelFontFamily`,
|
|
191
|
+
`labelFontWeight`. Run fields: `shape`, `crop`, `cropPadding`, `keepRaw`,
|
|
192
|
+
`scrollOffset`, `manifestPath`, `reportPath`.
|
|
193
|
+
|
|
194
|
+
## In CI
|
|
195
|
+
|
|
196
|
+
The annotated PNG overwrites the screenshot Cypress took, so it is what
|
|
197
|
+
`upload-artifact` collects and what `cypress run --record` sends to Cypress
|
|
198
|
+
Cloud. There is no Cloud API involved and nothing extra to configure.
|
|
199
|
+
|
|
200
|
+
Each run also writes `out/cypress/annotations.json`, one record per annotated
|
|
201
|
+
screenshot, and on GitHub Actions appends a table of it to the job summary.
|
|
202
|
+
|
|
160
203
|
## What this does not do
|
|
161
204
|
|
|
162
|
-
No LLM is involved in either path above, and no API key is needed.
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
install nothing by default. See `DOCS/pipeline.md` in the package.
|
|
205
|
+
No LLM is involved in either path above, and no API key is needed. For pointing
|
|
206
|
+
a model at a page so it works out what is wrong before anything gets drawn, see
|
|
207
|
+
the separate `annotate-agent` package.
|
package/DOCS/pipeline.md
DELETED
|
@@ -1,308 +0,0 @@
|
|
|
1
|
-
# The Playwright pipeline and the reasoning layer
|
|
2
|
-
|
|
3
|
-
Two things ship in `cypress-annotate` that the Cypress plugin never touches: a
|
|
4
|
-
general Playwright-driven annotation pipeline, and a Claude-based reasoning layer
|
|
5
|
-
that looks at a page and works out *what's* wrong before drawing the box. This
|
|
6
|
-
document covers both. For the Cypress plugin, see the [README](../README.md).
|
|
7
|
-
|
|
8
|
-
Both are optional. `playwright` and `@anthropic-ai/sdk` are optional
|
|
9
|
-
`peerDependencies`, so install whichever your own usage calls into:
|
|
10
|
-
|
|
11
|
-
```bash
|
|
12
|
-
npm install playwright @anthropic-ai/sdk
|
|
13
|
-
npx playwright install chromium
|
|
14
|
-
```
|
|
15
|
-
|
|
16
|
-
The repo's own suites for this half:
|
|
17
|
-
|
|
18
|
-
```bash
|
|
19
|
-
npm run verify # coordinate-pipeline alignment suite, 20 cases
|
|
20
|
-
npm run verify:image # driver-agnostic (MCP) alignment suite, 10 cases
|
|
21
|
-
npm run test:reasoner # Claude reasoning request/parse suite, 12 cases, no API key needed
|
|
22
|
-
npm run demo # end-to-end on the buggy fixture, using recorded findings
|
|
23
|
-
npm run annotate -- --url https://example.com --selector 'a' --label 'Broken link'
|
|
24
|
-
```
|
|
25
|
-
|
|
26
|
-
## API
|
|
27
|
-
|
|
28
|
-
```ts
|
|
29
|
-
import { captureAnnotated } from 'cypress-annotate';
|
|
30
|
-
|
|
31
|
-
const result = await captureAnnotated('https://example.com', [
|
|
32
|
-
{ selector: '#submit', label: 'Misaligned by 12px', shape: 'box' },
|
|
33
|
-
{ selector: ['iframe#checkout', '.pay-button'], label: 'Overlaps' },
|
|
34
|
-
], { mode: 'element', cropPadding: 80, style: { dimOutside: 0.5 } });
|
|
35
|
-
|
|
36
|
-
await writeFile('bug.png', result.image);
|
|
37
|
-
console.log(result.elements, result.drawnRects, result.warnings);
|
|
38
|
-
```
|
|
39
|
-
|
|
40
|
-
`captureAnnotated()` is the only export in the package that launches a browser,
|
|
41
|
-
and it imports playwright lazily on first call. `annotateImage`, `runBugHunt`,
|
|
42
|
-
and `findingsToAnnotations` all work with no browser installed.
|
|
43
|
-
|
|
44
|
-
Or drive an existing Playwright page directly with
|
|
45
|
-
`annotate(page, annotations, options)`. That's the entry point the reasoning step
|
|
46
|
-
uses, since the agent already has a page open.
|
|
47
|
-
|
|
48
|
-
**Modes.** `viewport` captures what's on screen, scrolling the first target into
|
|
49
|
-
view first. `fullPage` captures the whole document. `element` captures full-page
|
|
50
|
-
then crops to the annotated region plus `cropPadding`, which is usually the most
|
|
51
|
-
useful thing to attach to a ticket.
|
|
52
|
-
|
|
53
|
-
**Result.** Alongside `image`, you get `rawImage` (same capture, same crop, no
|
|
54
|
-
overlay), the measured `elements`, the final `drawnRects` in image pixels, and
|
|
55
|
-
`warnings`.
|
|
56
|
-
|
|
57
|
-
**Warnings worth acting on:** selector matched more than one element, element has
|
|
58
|
-
a zero-sized box, element is outside the viewport, element is fixed during a
|
|
59
|
-
full-page capture.
|
|
60
|
-
|
|
61
|
-
### Annotating an image somebody else captured
|
|
62
|
-
|
|
63
|
-
`annotateImage()` is the one export here that never opens a browser and knows
|
|
64
|
-
nothing about any driver. Give it a PNG, the page metrics that were true when it
|
|
65
|
-
was taken, and rectangles in CSS pixels. The Cypress plugin and the MCP path both
|
|
66
|
-
go through it, and so can anything else that can produce those three things.
|
|
67
|
-
|
|
68
|
-
```ts
|
|
69
|
-
import { annotateImage } from 'cypress-annotate';
|
|
70
|
-
import { readFile, writeFile } from 'node:fs/promises';
|
|
71
|
-
|
|
72
|
-
const result = await annotateImage(
|
|
73
|
-
await readFile('shot.png'),
|
|
74
|
-
[{
|
|
75
|
-
// Viewport-relative CSS px, straight from getBoundingClientRect().
|
|
76
|
-
viewportRect: { x: 100, y: 100, width: 320, height: 90 },
|
|
77
|
-
label: 'Apply button escapes its card',
|
|
78
|
-
}],
|
|
79
|
-
{
|
|
80
|
-
metrics: {
|
|
81
|
-
devicePixelRatio: 2,
|
|
82
|
-
scrollX: 0,
|
|
83
|
-
scrollY: 0,
|
|
84
|
-
viewportWidth: 1280,
|
|
85
|
-
viewportHeight: 800,
|
|
86
|
-
documentWidth: 1280,
|
|
87
|
-
documentHeight: 4200,
|
|
88
|
-
},
|
|
89
|
-
capture: 'viewport',
|
|
90
|
-
},
|
|
91
|
-
);
|
|
92
|
-
|
|
93
|
-
await writeFile('annotated.png', result.image);
|
|
94
|
-
console.log(result.scale, result.drawnRects, result.warnings);
|
|
95
|
-
```
|
|
96
|
-
|
|
97
|
-
The argument list is positional, `(screenshot, targets, options)`, and the
|
|
98
|
-
metrics are the four dimensions plus scroll and dpr. Both are easy to get wrong
|
|
99
|
-
from the type names alone, which is why the example is here.
|
|
100
|
-
|
|
101
|
-
`devicePixelRatio` is treated as a hint rather than a fact. `resolveScale()`
|
|
102
|
-
compares the image against the viewport it was supposed to cover and uses the
|
|
103
|
-
measured ratio when the two disagree, because a screenshot that was downscaled
|
|
104
|
-
behind your back is exactly what makes a box drift. Whatever it settles on comes
|
|
105
|
-
back as `result.scale`, and a mismatch lands in `result.warnings`.
|
|
106
|
-
|
|
107
|
-
Set `isFixed: true` on a target whose element is `position: fixed`, so it gets no
|
|
108
|
-
scroll correction. Pass `documentRect` yourself only when the viewport rect and
|
|
109
|
-
the scroll offset are not enough to derive it.
|
|
110
|
-
|
|
111
|
-
### The region fallback
|
|
112
|
-
|
|
113
|
-
When no element corresponds to the problem (blank space where something should
|
|
114
|
-
be), a target can be a rectangle in percentages instead:
|
|
115
|
-
|
|
116
|
-
```ts
|
|
117
|
-
{ region: { xPct: 10, yPct: 50, widthPct: 30, heightPct: 8, basis: 'viewport' } }
|
|
118
|
-
```
|
|
119
|
-
|
|
120
|
-
`basis: 'viewport'` percentages are relative to the scroll position they were
|
|
121
|
-
measured at, which is why the hunt never scrolls between measuring and drawing.
|
|
122
|
-
`basis: 'document'` is absolute. Both paths are covered by the alignment suite
|
|
123
|
-
and land on the same pixels the selector path does.
|
|
124
|
-
|
|
125
|
-
## The reasoning step
|
|
126
|
-
|
|
127
|
-
Point it at a URL. That's the whole invocation:
|
|
128
|
-
|
|
129
|
-
```bash
|
|
130
|
-
npm run find-bugs -- --url https://news.ycombinator.com
|
|
131
|
-
```
|
|
132
|
-
|
|
133
|
-
A ticket is optional. With one, findings are judged against it. Without one, the
|
|
134
|
-
page is judged on its own terms.
|
|
135
|
-
|
|
136
|
-
```bash
|
|
137
|
-
npm run find-bugs -- --url https://your.app/checkout \
|
|
138
|
-
--ticket "CHK-482: customer reviews the order and places it"
|
|
139
|
-
```
|
|
140
|
-
|
|
141
|
-
**The problem this solves.** A model cannot reliably invent a CSS selector by
|
|
142
|
-
looking at a screenshot. So it is never asked to. Before the screenshot is taken,
|
|
143
|
-
every candidate element is tagged with a `data-annot-ref` attribute and listed
|
|
144
|
-
for the model with its tag, accessible name and bounding box:
|
|
145
|
-
|
|
146
|
-
```
|
|
147
|
-
[e17] <button> "Apply" @ 1090,642 87x39 — #promo-apply
|
|
148
|
-
[e24] <span> "£244.99" @ 1262,1046 57x22 — #grand-total-value
|
|
149
|
-
```
|
|
150
|
-
|
|
151
|
-
The model returns a handle, which maps back to a selector guaranteed to resolve
|
|
152
|
-
because we injected it. The readable selector (`#promo-apply`) still goes in the
|
|
153
|
-
ticket. Those bounding boxes are also the model's best evidence for geometric
|
|
154
|
-
bugs, since overlapping boxes and boxes that run past a parent show up in the
|
|
155
|
-
numbers as clearly as in the pixels.
|
|
156
|
-
|
|
157
|
-
**Structured output.** Findings come back through a strict tool schema:
|
|
158
|
-
|
|
159
|
-
```json
|
|
160
|
-
{ "description": "...", "evidence": "...", "severity": "blocker", "ref": "e17", "region": null }
|
|
161
|
-
```
|
|
162
|
-
|
|
163
|
-
Severity drives the annotation colour. Every finding gets a cropped, dimmed
|
|
164
|
-
close-up, and each pass also produces an overview with numbered markers, because
|
|
165
|
-
full-sentence labels on an overview cover the rest of the page.
|
|
166
|
-
|
|
167
|
-
**Three resolution paths, all tested:** a valid handle resolves to its injected
|
|
168
|
-
selector; a handle the model invented falls back to the region it supplied; a
|
|
169
|
-
finding with neither is reported as unresolved rather than silently dropped.
|
|
170
|
-
|
|
171
|
-
**Viewport sweep.** `runBugHunt` works down the page one viewport at a time,
|
|
172
|
-
overlapping passes by 10%. A pass is the unit of correctness: the inventory, the
|
|
173
|
-
screenshot the model reasons over, and any percentage region it returns all share
|
|
174
|
-
one scroll position. Findings are de-duplicated across passes by readable
|
|
175
|
-
selector, since handles are re-issued each pass.
|
|
176
|
-
|
|
177
|
-
## Using this from a live agent session
|
|
178
|
-
|
|
179
|
-
When Claude is already driving a real browser, say a Playwright MCP or Chrome
|
|
180
|
-
DevTools MCP session that is logged in, the thing to avoid is opening a second
|
|
181
|
-
one. A second browser would not share the session and would land on the SSO login
|
|
182
|
-
page, so this path never opens one.
|
|
183
|
-
|
|
184
|
-
Three pieces:
|
|
185
|
-
|
|
186
|
-
- **[src/browser/measure-target.js](../src/browser/measure-target.js)** is a
|
|
187
|
-
snippet to paste into whichever `evaluate` tool is live. Edit one line, run it,
|
|
188
|
-
get `{ metrics, targets }` back. Both backends can write the result straight to
|
|
189
|
-
a file (`filename` on Playwright MCP, `filePath` on Chrome DevTools MCP).
|
|
190
|
-
- **`npm run annotate-shot`** takes a PNG plus that JSON and draws the boxes. It
|
|
191
|
-
never opens a browser and knows nothing about either backend.
|
|
192
|
-
- **[skills/annotate-bug/SKILL.md](../skills/annotate-bug/SKILL.md)** is the
|
|
193
|
-
protocol written as a skill, with the exact tool calls for both backends. Copy
|
|
194
|
-
it into `.claude/skills/`.
|
|
195
|
-
|
|
196
|
-
```bash
|
|
197
|
-
npm run annotate-shot -- --image shot.png --measurement measure.json \
|
|
198
|
-
--label "Apply button escapes its card" --out annotated.png --crop
|
|
199
|
-
```
|
|
200
|
-
|
|
201
|
-
**The ordering rule that makes it work:** measure, then screenshot, with no
|
|
202
|
-
navigation, scroll, or click in between. A rectangle is only valid for the scroll
|
|
203
|
-
position it was measured at. Everything else the pipeline can recover from. This
|
|
204
|
-
it cannot.
|
|
205
|
-
|
|
206
|
-
**The scale is measured on every capture.** Both backends can hand back a resized
|
|
207
|
-
image, Playwright MCP via `scale: "css"` and Chrome DevTools MCP via
|
|
208
|
-
`--screenshot-max-width`, so `devicePixelRatio` describes the browser and says
|
|
209
|
-
nothing about the file on disk. `resolveScale()` compares the image against the
|
|
210
|
-
viewport it was supposed to cover, uses the measured ratio when the two disagree,
|
|
211
|
-
and prints a warning. `npm run verify:image` proves this holds at scales from 0.5
|
|
212
|
-
to 2.0, including a case downscaled to 900px from a 2560px capture.
|
|
213
|
-
|
|
214
|
-
If there is no element to point at (blank space where something should render),
|
|
215
|
-
`--region x,y,w,h` takes percentages of the viewport instead.
|
|
216
|
-
|
|
217
|
-
## How the reasoning step is verified
|
|
218
|
-
|
|
219
|
-
The live call has been run against the real API (see below). It is also tested
|
|
220
|
-
without credentials, so the suite stays runnable and free. `npm run test:reasoner`
|
|
221
|
-
drives `ClaudeReasoner` through a stubbed `fetch` that returns a real SSE stream,
|
|
222
|
-
covering request construction (model, strict tool schema, adaptive thinking, the
|
|
223
|
-
image and inventory blocks), stream handling, tool-use parsing, all three
|
|
224
|
-
resolution paths, refusals, and malformed model output. 12 checks.
|
|
225
|
-
|
|
226
|
-
`npm run demo` then runs the whole pipeline against
|
|
227
|
-
[fixtures/buggy-checkout.html](../fixtures/buggy-checkout.html), a page with five
|
|
228
|
-
deliberate bugs, using recorded findings in place of the model. The recorded
|
|
229
|
-
findings name real CSS selectors, which the replay translates into the handles
|
|
230
|
-
the model would have returned, so the resolution path under test is identical to
|
|
231
|
-
the live one. Two of the seven entries are deliberate fallback tests.
|
|
232
|
-
|
|
233
|
-
**To run it live**, copy `.env.example` to `.env` and put your key in it:
|
|
234
|
-
|
|
235
|
-
```bash
|
|
236
|
-
cp .env.example .env
|
|
237
|
-
# ANTHROPIC_API_KEY=sk-ant-...
|
|
238
|
-
|
|
239
|
-
npm run find-bugs -- --url https://your.app/checkout --ticket "CHK-482: ..."
|
|
240
|
-
```
|
|
241
|
-
|
|
242
|
-
`.env` is read by `scripts/find-bugs.ts` via `process.loadEnvFile` and is
|
|
243
|
-
gitignored. Exporting `ANTHROPIC_API_KEY` in your shell works too, as does an
|
|
244
|
-
`ant auth login` profile, and the SDK resolves credentials in that order. It
|
|
245
|
-
resolves them lazily, at request time rather than when the client is constructed,
|
|
246
|
-
so a missing key surfaces on the first request. The CLI catches that and prints
|
|
247
|
-
setup instructions instead of a stack trace.
|
|
248
|
-
|
|
249
|
-
The library itself never reads `.env`. Only the CLI does, so importing
|
|
250
|
-
`annotate()` or `ClaudeReasoner` into your own program leaves credential handling
|
|
251
|
-
to you.
|
|
252
|
-
|
|
253
|
-
### What the first live runs showed
|
|
254
|
-
|
|
255
|
-
On [fixtures/buggy-checkout.html](../fixtures/buggy-checkout.html) with **no
|
|
256
|
-
ticket**, Claude found 4 of the 5 planted bugs, every selector resolved, and
|
|
257
|
-
nothing was unresolved. It missed the `#place-order` button overlapping the terms
|
|
258
|
-
text.
|
|
259
|
-
|
|
260
|
-
The evidence it gave is the part that matters, because it shows the design
|
|
261
|
-
premise holding up. It reasons from the bounding boxes as much as from the
|
|
262
|
-
pixels:
|
|
263
|
-
|
|
264
|
-
> `#postcode-field`: "e10/e11/e12 start at x=419 while all other fields (e3, e7,
|
|
265
|
-
> e9, e13) start at x=385, and the input is narrower (476 vs 510)."
|
|
266
|
-
|
|
267
|
-
Two real bugs surfaced from pointing it at Hacker News, both now fixed and
|
|
268
|
-
covered by the alignment suite:
|
|
269
|
-
|
|
270
|
-
- **Quirks mode.** HN has no doctype, so `documentElement.clientHeight` returns
|
|
271
|
-
the *content* height (1214) instead of the viewport height (900). The sweep
|
|
272
|
-
concluded the page fit on one screen, and percentage regions were measured
|
|
273
|
-
against the wrong height. Metrics now use `window.innerWidth/innerHeight`, and
|
|
274
|
-
[fixtures/quirks-page.html](../fixtures/quirks-page.html) covers it.
|
|
275
|
-
- **Stale handles.** Each pass restarts its counter at `e1`, but the previous
|
|
276
|
-
pass's attributes were still in the DOM, so `[data-annot-ref="e5"]` could match
|
|
277
|
-
two elements and annotate the wrong one. The collector now clears stale handles
|
|
278
|
-
before tagging.
|
|
279
|
-
|
|
280
|
-
Both surfaced from the pipeline reporting its own warnings, which is the argument
|
|
281
|
-
for keeping `warnings` in the report.
|
|
282
|
-
|
|
283
|
-
## Known limits
|
|
284
|
-
|
|
285
|
-
- Findings vary between runs on the same page, as you'd expect. Two runs of the
|
|
286
|
-
Hacker News front page returned three findings and one.
|
|
287
|
-
- Judgement is not calibrated. The missing-domain finding on Hacker News is
|
|
288
|
-
arguably a false positive, since HN legitimately shows no domain for text
|
|
289
|
-
posts. The model considered that and reported it anyway.
|
|
290
|
-
- `element` mode captures full-page and then crops, which is wasteful on very
|
|
291
|
-
long pages. Fine at this stage, worth revisiting if it becomes a bottleneck.
|
|
292
|
-
- The inventory covers the main document and same-origin iframes. Cross-origin
|
|
293
|
-
frames are skipped, so bugs inside them will not be found.
|
|
294
|
-
- Overview labels can still overlap each other when findings are close together.
|
|
295
|
-
|
|
296
|
-
## What's next here
|
|
297
|
-
|
|
298
|
-
Selector accuracy looks like the solved part. Across the live runs so far, every
|
|
299
|
-
handle Claude picked resolved to the element it described, and no real page has
|
|
300
|
-
needed the region fallback yet.
|
|
301
|
-
|
|
302
|
-
The open question has shifted to **judgement**, meaning precision and recall.
|
|
303
|
-
Claude missed the button and text overlap on the fixture, and its Hacker News
|
|
304
|
-
finding is arguably a false positive. Worth running against pages with known
|
|
305
|
-
defects and counting both the misses and the false alarms. `report.json` already
|
|
306
|
-
carries what that needs: `selector` is what Claude chose, `resolvedTo` is what
|
|
307
|
-
got annotated, `warnings` flags ambiguity, and `unresolved` lists what it could
|
|
308
|
-
not point at.
|
package/dist/annotate.d.ts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { Page } from 'playwright';
|
|
2
|
-
import type { Annotation, AnnotateOptions, AnnotateResult } from './types.js';
|
|
3
|
-
/**
|
|
4
|
-
* Playwright-driven annotation: position the page, measure, capture, and hand
|
|
5
|
-
* the pixels to `annotateImage`, which owns all the coordinate maths and is
|
|
6
|
-
* shared with the driver-agnostic path.
|
|
7
|
-
*/
|
|
8
|
-
export declare function annotate(page: Page, annotations: Annotation[], options?: AnnotateOptions): Promise<AnnotateResult>;
|
|
9
|
-
//# sourceMappingURL=annotate.d.ts.map
|
package/dist/annotate.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"annotate.d.ts","sourceRoot":"","sources":["../src/annotate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,cAAc,EAAmB,MAAM,YAAY,CAAC;AAE/F;;;;GAIG;AACH,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,IAAI,EACV,WAAW,EAAE,UAAU,EAAE,EACzB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC,CA+EzB"}
|