cypress-annotate 0.3.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/AGENTS.md +40 -28
  2. package/CHANGELOG.md +142 -0
  3. package/DOCS/images/crop.png +0 -0
  4. package/DOCS/images/failure-hook.png +0 -0
  5. package/DOCS/internals.md +151 -0
  6. package/README.md +104 -329
  7. package/dist/annotate-image.d.ts +5 -0
  8. package/dist/annotate-image.d.ts.map +1 -1
  9. package/dist/annotate-image.js +16 -2
  10. package/dist/cypress/commands.d.ts +6 -1
  11. package/dist/cypress/commands.d.ts.map +1 -1
  12. package/dist/cypress/commands.js +13 -6
  13. package/dist/cypress/config.d.ts +42 -0
  14. package/dist/cypress/config.d.ts.map +1 -0
  15. package/dist/cypress/config.js +102 -0
  16. package/dist/cypress/failure-hook.d.ts.map +1 -1
  17. package/dist/cypress/failure-hook.js +10 -3
  18. package/dist/cypress/failure-report.d.ts +6 -0
  19. package/dist/cypress/failure-report.d.ts.map +1 -1
  20. package/dist/cypress/failure-report.js +14 -4
  21. package/dist/cypress/task.d.ts +36 -2
  22. package/dist/cypress/task.d.ts.map +1 -1
  23. package/dist/cypress/task.js +60 -5
  24. package/dist/draw.d.ts +4 -0
  25. package/dist/draw.d.ts.map +1 -1
  26. package/dist/draw.js +7 -4
  27. package/dist/index.d.ts +8 -28
  28. package/dist/index.d.ts.map +1 -1
  29. package/dist/index.js +7 -37
  30. package/dist/types.d.ts +6 -72
  31. package/dist/types.d.ts.map +1 -1
  32. package/package.json +10 -39
  33. package/skills/cypress-annotate/SKILL.md +51 -9
  34. package/DOCS/pipeline.md +0 -308
  35. package/dist/annotate.d.ts +0 -9
  36. package/dist/annotate.d.ts.map +0 -1
  37. package/dist/annotate.js +0 -82
  38. package/dist/browser/collect-elements.js +0 -158
  39. package/dist/browser/measure-target.js +0 -69
  40. package/dist/claude-reasoner.d.ts +0 -17
  41. package/dist/claude-reasoner.d.ts.map +0 -1
  42. package/dist/claude-reasoner.js +0 -192
  43. package/dist/cli.d.ts +0 -2
  44. package/dist/cli.d.ts.map +0 -1
  45. package/dist/cli.js +0 -74
  46. package/dist/cypress/failure-explainer.d.ts +0 -45
  47. package/dist/cypress/failure-explainer.d.ts.map +0 -1
  48. package/dist/cypress/failure-explainer.js +0 -123
  49. package/dist/finder.d.ts +0 -68
  50. package/dist/finder.d.ts.map +0 -1
  51. package/dist/finder.js +0 -64
  52. package/dist/inventory.d.ts +0 -33
  53. package/dist/inventory.d.ts.map +0 -1
  54. package/dist/inventory.js +0 -88
  55. package/dist/measure.d.ts +0 -11
  56. package/dist/measure.d.ts.map +0 -1
  57. package/dist/measure.js +0 -145
  58. package/dist/pipeline.d.ts +0 -60
  59. package/dist/pipeline.d.ts.map +0 -1
  60. package/dist/pipeline.js +0 -115
  61. package/skills/annotate-bug/SKILL.md +0 -121
  62. package/src/browser/collect-elements.js +0 -158
  63. package/src/browser/measure-target.js +0 -69
package/dist/pipeline.js DELETED
@@ -1,115 +0,0 @@
1
- import { annotate } from './annotate.js';
2
- import { findingsToAnnotations, gatherReasonerInput, } from './finder.js';
3
- import { readPageMetrics } from './measure.js';
4
- /**
5
- * Work down the page one viewport at a time, reasoning about each and
6
- * annotating whatever comes back.
7
- *
8
- * A pass is the unit of correctness here: the inventory, the screenshot the
9
- * model reasons over, and any percentage region it returns must all share one
10
- * scroll position. Mixing scroll positions between those three is what makes
11
- * region fallbacks land in the wrong place, so each pass scrolls, settles, and
12
- * only then measures.
13
- */
14
- export async function runBugHunt(page, options) {
15
- const maxPasses = options.maxPasses ?? 6;
16
- // Reuse the one definition of these metrics rather than re-deriving them:
17
- // getting the viewport height wrong here silently collapses the sweep to a
18
- // single pass on any quirks-mode page.
19
- const { viewportHeight, documentHeight } = await readPageMetrics(page);
20
- // Overlap each pass by 10% so a bug straddling a boundary is fully visible in
21
- // at least one of them.
22
- const step = Math.max(1, Math.floor(viewportHeight * 0.9));
23
- const positions = [];
24
- for (let y = 0; positions.length < maxPasses; y += step) {
25
- const clamped = Math.min(y, Math.max(0, documentHeight - viewportHeight));
26
- if (positions.length > 0 && clamped === positions[positions.length - 1])
27
- break;
28
- positions.push(clamped);
29
- if (clamped + viewportHeight >= documentHeight)
30
- break;
31
- }
32
- const annotated = [];
33
- const unresolvedByDescription = new Map();
34
- const passes = [];
35
- const inventoryBySelector = new Map();
36
- const seenKeys = new Set();
37
- let findingNumber = 0;
38
- for (const [passIndex, scrollY] of positions.entries()) {
39
- await page.evaluate((y) => window.scrollTo(0, y), scrollY);
40
- await page.waitForTimeout(120);
41
- const input = await gatherReasonerInput(page, options.ticket, true);
42
- for (const item of input.inventory) {
43
- if (!inventoryBySelector.has(item.reportSelector)) {
44
- inventoryBySelector.set(item.reportSelector, item);
45
- }
46
- }
47
- const findings = await options.reasoner.propose(input);
48
- const resolution = findingsToAnnotations(findings, input.inventory);
49
- for (const entry of resolution.unresolved) {
50
- // An element below the fold on this pass may resolve on the next one, so
51
- // key by description and reconcile against the annotated set at the end.
52
- if (!unresolvedByDescription.has(entry.finding.description)) {
53
- unresolvedByDescription.set(entry.finding.description, entry);
54
- }
55
- }
56
- // A bug visible in the overlap between two passes gets reported twice.
57
- const fresh = resolution.resolved
58
- .filter((entry) => !seenKeys.has(entry.key))
59
- .map((entry) => ({ entry, number: ++findingNumber }));
60
- for (const { entry } of fresh)
61
- seenKeys.add(entry.key);
62
- let overview = null;
63
- if (fresh.length > 0) {
64
- // Full sentences would cover the rest of the page here, so the overview
65
- // carries a marker and the close-up carries the description.
66
- const result = await annotate(page, fresh.map(({ entry, number }) => ({
67
- ...entry.annotation,
68
- label: `#${number} ${entry.finding.severity}`,
69
- })), { mode: 'viewport', scrollIntoView: false });
70
- overview = result.image;
71
- }
72
- passes.push({
73
- index: passIndex,
74
- scrollY,
75
- inventorySize: input.inventory.length,
76
- findingCount: fresh.length,
77
- overview,
78
- });
79
- if (options.perFinding ?? true) {
80
- for (const { entry, number } of fresh) {
81
- const labelled = {
82
- ...entry.annotation,
83
- label: `#${number} ${entry.finding.severity}: ${entry.finding.description}`,
84
- };
85
- const result = await annotate(page, [labelled], {
86
- mode: 'element',
87
- cropPadding: options.cropPadding ?? 90,
88
- // Staying at this pass's scroll position keeps percentage regions
89
- // meaningful; annotate() converts to document space from here.
90
- scrollToTop: false,
91
- style: { dimOutside: 0.45 },
92
- });
93
- annotated.push({
94
- number,
95
- finding: entry.finding,
96
- annotation: labelled,
97
- resolvedTo: result.elements[0]?.selector ?? 'unknown',
98
- reportSelector: entry.item?.reportSelector ?? null,
99
- pass: passIndex,
100
- image: result.image,
101
- drawnRect: result.drawnRects[0],
102
- warnings: result.warnings,
103
- });
104
- }
105
- }
106
- }
107
- const annotatedDescriptions = new Set(annotated.map((a) => a.finding.description));
108
- const unresolved = Array.from(unresolvedByDescription.values()).filter((entry) => !annotatedDescriptions.has(entry.finding.description));
109
- return {
110
- annotated,
111
- unresolved,
112
- passes,
113
- inventory: Array.from(inventoryBySelector.values()),
114
- };
115
- }
@@ -1,121 +0,0 @@
1
- ---
2
- name: annotate-bug
3
- description: Draw an accurate box on a screenshot of the live page to show exactly where a visual bug is. Use after finding a layout or rendering defect during browser testing, when the bug report needs to point at a specific element rather than attach a raw screenshot. Works with whichever browser MCP backend is connected - Playwright MCP or Chrome DevTools MCP.
4
- ---
5
-
6
- # Annotating a bug on a live page
7
-
8
- Turns "the Apply button escapes its card" into a screenshot with a box drawn
9
- around that exact element.
10
-
11
- This skill does **not** open a browser. It annotates a screenshot that the
12
- browser backend already connected to this session has captured. That matters:
13
- the live session is the one that is logged in, and a separately launched browser
14
- would land on the SSO login page instead of the page under test.
15
-
16
- ## Setup
17
-
18
- Set `ANNOTATE_REPO` to wherever the tool is checked out (it needs
19
- `npm install` once):
20
-
21
- ```
22
- ANNOTATE_REPO=/path/to/cypress-annotate
23
- ```
24
-
25
- ## The protocol
26
-
27
- Four steps, in this order. The ordering is the whole discipline: a rectangle is
28
- only valid for the scroll position it was measured at.
29
-
30
- ### 1. Choose the element
31
-
32
- Pick the single element that *is* wrong - not its container, and not the element
33
- it collides with. If a page snapshot gave you an element ref, resolve it to a CSS
34
- selector; prefer `#id` or `[data-testid=…]` over a structural path.
35
-
36
- ### 2. Measure it
37
-
38
- Read `$ANNOTATE_REPO/src/browser/measure-target.js`, replace the `SELECTORS`
39
- line with your selector(s), and evaluate the whole function in the page. Save the
40
- result to a file.
41
-
42
- **Playwright MCP:**
43
- ```
44
- browser_evaluate({ function: "<the edited snippet>", filename: "/abs/path/measure.json" })
45
- ```
46
-
47
- **Chrome DevTools MCP:**
48
- ```
49
- evaluate_script({ pageId: <id>, function: "<the edited snippet>", filePath: "/abs/path/measure.json" })
50
- ```
51
-
52
- Check the result: each target has `found: true` and a `rect`. If `found` is
53
- false, the selector is wrong - fix it rather than proceeding, or fall back to a
54
- region (below). If `matchCount` is above 1, the selector is ambiguous and the
55
- first match gets annotated, which may not be the one you mean.
56
-
57
- ### 3. Screenshot immediately
58
-
59
- Take the screenshot **without navigating, scrolling, clicking, or waiting for a
60
- re-render in between**. Anything that moves the page invalidates the rectangles
61
- from step 2 and the box will be drawn in the wrong place.
62
-
63
- **Playwright MCP:**
64
- ```
65
- browser_take_screenshot({ filename: "/abs/path/shot.png" })
66
- ```
67
-
68
- **Chrome DevTools MCP:**
69
- ```
70
- take_screenshot({ pageId: <id>, filePath: "/abs/path/shot.png", format: "png" })
71
- ```
72
-
73
- Viewport capture is the default and is what you want. If you pass `fullPage`,
74
- pass `--capture fullPage` in step 4 as well, or every box below the fold will be
75
- misplaced.
76
-
77
- ### 4. Draw the box
78
-
79
- ```bash
80
- cd $ANNOTATE_REPO && npm run annotate-shot -- \
81
- --image /abs/path/shot.png \
82
- --measurement /abs/path/measure.json \
83
- --label "Apply button escapes its card" \
84
- --out /abs/path/annotated.png
85
- ```
86
-
87
- One `--label` per target, in the same order as `SELECTORS`. Useful extras:
88
-
89
- - `--crop` - close-up of the element with the rest dimmed. Best for a ticket.
90
- - `--capture fullPage` - required if the screenshot was a full-page capture.
91
- - `--shape arrow` - point at it instead of boxing it.
92
- - `--color '#FF6A00'` - e.g. colour by severity.
93
-
94
- Then attach `annotated.png` to the report and say what is wrong in words as well.
95
-
96
- ## When there is no element to point at
97
-
98
- If the defect is *absent* content - blank space where something should render -
99
- there is no selector to measure. Skip step 2 and give a rectangle as percentages
100
- of the viewport:
101
-
102
- ```bash
103
- npm run annotate-shot -- --image shot.png --region 10,50,30,8 \
104
- --label "Delivery estimate missing" --out annotated.png
105
- ```
106
-
107
- `--region` is `x,y,width,height` in percent. Use it only as a fallback: a
108
- selector is measured, a region is guessed.
109
-
110
- ## Checks worth reading
111
-
112
- The command prints warnings. Two are worth acting on:
113
-
114
- - *"matched N elements; annotated the first"* - the selector is ambiguous. Make
115
- it specific and re-run.
116
- - *"the image was probably rescaled"* - not a problem. Both backends can resize
117
- screenshots, so the true scale is measured from the image rather than trusted
118
- from `devicePixelRatio`. The box is still correct.
119
-
120
- If the box lands in the wrong place, the cause is almost always that the page
121
- moved between steps 2 and 3. Redo both, back to back.
@@ -1,158 +0,0 @@
1
- // This file is read as text and evaluated inside the page - it is never
2
- // imported or bundled. Keeping it out of the TypeScript build is deliberate:
3
- // esbuild's keepNames transform injects a `__name` helper into any function it
4
- // compiles, and that helper does not exist in the browser, so a serialized
5
- // TypeScript function throws `__name is not defined` on evaluation.
6
- //
7
- // The whole file is therefore a single arrow-function expression.
8
- //
9
- // Tags every candidate element with a ref attribute and returns a description
10
- // of it. Tagging rather than deriving a selector afterwards is deliberate: a
11
- // selector we injected always resolves, whereas a path built from classes can
12
- // be invalidated by anything that re-renders between measuring and annotating.
13
-
14
- (function (options) {
15
- var prefix = options.prefix;
16
- var refAttr = options.refAttr;
17
- var minSize = options.minSize;
18
- var viewportOnly = options.viewportOnly;
19
-
20
- var INTERACTIVE = ['a', 'button', 'input', 'select', 'textarea', 'label', 'summary', 'option'];
21
- var STRUCTURAL = ['header', 'nav', 'main', 'footer', 'aside', 'form', 'table', 'img', 'svg', 'video', 'dialog'];
22
- var HEADING = /^h[1-6]$/;
23
- var testAttrs = ['data-testid', 'data-test-id', 'data-test', 'data-cy', 'data-qa'];
24
-
25
- function readableSelector(el) {
26
- // Prefer whatever a developer would actually paste into devtools.
27
- var id = el.getAttribute('id');
28
- if (id && /^[A-Za-z][\w-]*$/.test(id) && el.ownerDocument.querySelectorAll('#' + id).length === 1) {
29
- return '#' + id;
30
- }
31
-
32
- for (var i = 0; i < testAttrs.length; i++) {
33
- var value = el.getAttribute(testAttrs[i]);
34
- if (value) {
35
- var candidate = '[' + testAttrs[i] + '="' + value + '"]';
36
- if (el.ownerDocument.querySelectorAll(candidate).length === 1) return candidate;
37
- }
38
- }
39
-
40
- var parts = [];
41
- var node = el;
42
- for (var depth = 0; node && depth < 4; depth++) {
43
- var tag = node.tagName.toLowerCase();
44
- if (tag === 'body' || tag === 'html') break;
45
-
46
- var nodeId = node.getAttribute('id');
47
- if (nodeId && /^[A-Za-z][\w-]*$/.test(nodeId)) {
48
- parts.unshift('#' + nodeId);
49
- break;
50
- }
51
-
52
- var parent = node.parentElement;
53
- var part = tag;
54
- if (parent) {
55
- var siblings = [];
56
- for (var c = 0; c < parent.children.length; c++) {
57
- if (parent.children[c].tagName === node.tagName) siblings.push(parent.children[c]);
58
- }
59
- if (siblings.length > 1) part += ':nth-of-type(' + (siblings.indexOf(node) + 1) + ')';
60
- }
61
- parts.unshift(part);
62
-
63
- var joined = parts.join(' > ');
64
- try {
65
- if (el.ownerDocument.querySelectorAll(joined).length === 1) return joined;
66
- } catch (e) {
67
- // An unusual tag name can produce an invalid selector; keep walking.
68
- }
69
- node = parent;
70
- }
71
- return parts.join(' > ') || el.tagName.toLowerCase();
72
- }
73
-
74
- function accessibleName(el) {
75
- var aria = el.getAttribute('aria-label');
76
- if (aria) return aria.trim();
77
- var alt = el.getAttribute('alt');
78
- if (alt) return alt.trim();
79
- var placeholder = el.getAttribute('placeholder');
80
- if (placeholder) return 'placeholder: ' + placeholder.trim();
81
- if (el.tagName === 'INPUT' && typeof el.value === 'string' && el.value) {
82
- return 'value: ' + el.value.trim();
83
- }
84
- return (el.textContent || '').replace(/\s+/g, ' ').trim().slice(0, 80);
85
- }
86
-
87
- // An element a developer deliberately named is worth listing even when it is
88
- // just a wrapper - a misaligned or overflowing container is exactly the kind
89
- // of target whose children, not itself, carry the text.
90
- function hasStableName(el) {
91
- if (el.hasAttribute('id')) return true;
92
- for (var i = 0; i < testAttrs.length; i++) {
93
- if (el.hasAttribute(testAttrs[i])) return true;
94
- }
95
- return false;
96
- }
97
-
98
- // Text-bearing leaf: has its own text but no child element that also has text.
99
- function isTextLeaf(el) {
100
- if (!(el.textContent || '').trim()) return false;
101
- for (var i = 0; i < el.children.length; i++) {
102
- if ((el.children[i].textContent || '').trim()) return false;
103
- }
104
- return true;
105
- }
106
-
107
- // Clear handles left by an earlier pass. Counters restart at 1 each time, so
108
- // without this a stale e5 from a previous scroll position still matches
109
- // [data-annot-ref="e5"] and the annotation can land on the wrong element.
110
- var stale = document.querySelectorAll('[' + refAttr + ']');
111
- for (var s = 0; s < stale.length; s++) stale[s].removeAttribute(refAttr);
112
-
113
- var results = [];
114
- var counter = 0;
115
- var all = document.querySelectorAll('*');
116
-
117
- for (var i = 0; i < all.length; i++) {
118
- var el = all[i];
119
- var tag = el.tagName.toLowerCase();
120
- if (tag === 'script' || tag === 'style' || tag === 'meta' || tag === 'link') continue;
121
-
122
- var style = getComputedStyle(el);
123
- if (style.visibility === 'hidden' || style.display === 'none') continue;
124
- if (parseFloat(style.opacity) < 0.05) continue;
125
-
126
- var rect = el.getBoundingClientRect();
127
- if (rect.width < minSize || rect.height < minSize) continue;
128
-
129
- var intersects =
130
- rect.bottom > 0 && rect.right > 0 &&
131
- rect.top < window.innerHeight && rect.left < window.innerWidth;
132
- if (viewportOnly && !intersects) continue;
133
-
134
- var interesting =
135
- INTERACTIVE.indexOf(tag) !== -1 ||
136
- STRUCTURAL.indexOf(tag) !== -1 ||
137
- HEADING.test(tag) ||
138
- el.hasAttribute('role') ||
139
- hasStableName(el) ||
140
- isTextLeaf(el);
141
- if (!interesting) continue;
142
-
143
- var ref = prefix + ++counter;
144
- el.setAttribute(refAttr, ref);
145
-
146
- results.push({
147
- ref: ref,
148
- reportSelector: readableSelector(el),
149
- tag: tag,
150
- role: el.getAttribute('role'),
151
- name: accessibleName(el),
152
- rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
153
- inViewport: intersects,
154
- });
155
- }
156
-
157
- return results;
158
- })
@@ -1,69 +0,0 @@
1
- // Paste this into whichever browser MCP is live, as the evaluate function:
2
- //
3
- // Playwright MCP browser_evaluate({ function: "<this>", filename: "measure.json" })
4
- // Chrome DevTools evaluate_script({ pageId, function: "<this>", filePath: "measure.json" })
5
- //
6
- // Edit the SELECTORS line, and nothing else. The result is the exact JSON that
7
- // `annotate-shot --measurement` expects.
8
- //
9
- // It deliberately does not scroll, click, or change anything: every rectangle it
10
- // returns is only valid for the scroll position the page is in right now, so
11
- // the screenshot must be taken immediately after this runs, with no navigation
12
- // in between.
13
-
14
- () => {
15
- const SELECTORS = ['#place-order'];
16
-
17
- const doc = document.documentElement;
18
- const body = document.body;
19
-
20
- const metrics = {
21
- devicePixelRatio: window.devicePixelRatio,
22
- scrollX: window.scrollX,
23
- scrollY: window.scrollY,
24
- // innerWidth/Height rather than documentElement.clientWidth/Height: on a
25
- // page with no doctype (quirks mode) the root element reports the content
26
- // size, not the viewport.
27
- viewportWidth: window.innerWidth,
28
- viewportHeight: window.innerHeight,
29
- documentWidth: Math.max(doc.scrollWidth, body ? body.scrollWidth : 0),
30
- documentHeight: Math.max(doc.scrollHeight, body ? body.scrollHeight : 0),
31
- };
32
-
33
- const targets = SELECTORS.map((selector) => {
34
- let matches;
35
- try {
36
- matches = document.querySelectorAll(selector);
37
- } catch (e) {
38
- return { selector, found: false, error: 'invalid selector: ' + e.message };
39
- }
40
- if (matches.length === 0) return { selector, found: false, error: 'no element matched' };
41
-
42
- const el = matches[0];
43
- const r = el.getBoundingClientRect();
44
-
45
- // An element inside a fixed subtree does not move with document scroll, so
46
- // it has no meaningful document-space position.
47
- let isFixed = false;
48
- for (let node = el; node; node = node.parentElement) {
49
- if (getComputedStyle(node).position === 'fixed') {
50
- isFixed = true;
51
- break;
52
- }
53
- }
54
-
55
- return {
56
- selector,
57
- found: true,
58
- matchCount: matches.length,
59
- tag: el.tagName.toLowerCase(),
60
- text: (el.textContent || '').replace(/\s+/g, ' ').trim().slice(0, 80),
61
- isFixed,
62
- rect: { x: r.x, y: r.y, width: r.width, height: r.height },
63
- inViewport:
64
- r.bottom > 0 && r.right > 0 && r.top < window.innerHeight && r.left < window.innerWidth,
65
- };
66
- });
67
-
68
- return { metrics, targets };
69
- }