cypress-annotate 0.4.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +40 -28
- package/CHANGELOG.md +174 -0
- package/DOCS/images/crop.png +0 -0
- package/DOCS/images/failure-hook.png +0 -0
- package/DOCS/internals.md +151 -0
- package/README.md +104 -352
- package/dist/annotate-image.d.ts +19 -2
- package/dist/annotate-image.d.ts.map +1 -1
- package/dist/annotate-image.js +55 -12
- package/dist/cypress/commands.d.ts +6 -1
- package/dist/cypress/commands.d.ts.map +1 -1
- package/dist/cypress/commands.js +20 -6
- package/dist/cypress/config.d.ts +42 -0
- package/dist/cypress/config.d.ts.map +1 -0
- package/dist/cypress/config.js +102 -0
- package/dist/cypress/failure-hook.d.ts.map +1 -1
- package/dist/cypress/failure-hook.js +17 -3
- package/dist/cypress/failure-report.d.ts +6 -0
- package/dist/cypress/failure-report.d.ts.map +1 -1
- package/dist/cypress/failure-report.js +14 -4
- package/dist/cypress/task.d.ts +39 -3
- package/dist/cypress/task.d.ts.map +1 -1
- package/dist/cypress/task.js +61 -5
- package/dist/draw.d.ts +4 -0
- package/dist/draw.d.ts.map +1 -1
- package/dist/draw.js +7 -4
- package/dist/index.d.ts +9 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -57
- package/dist/types.d.ts +13 -69
- package/dist/types.d.ts.map +1 -1
- package/package.json +11 -43
- package/skills/cypress-annotate/SKILL.md +62 -9
- package/DOCS/pipeline.md +0 -308
- package/dist/annotate.d.ts +0 -9
- package/dist/annotate.d.ts.map +0 -1
- package/dist/annotate.js +0 -82
- package/dist/browser/collect-elements.js +0 -158
- package/dist/browser/measure-target.js +0 -69
- package/dist/claude-reasoner.d.ts +0 -17
- package/dist/claude-reasoner.d.ts.map +0 -1
- package/dist/claude-reasoner.js +0 -192
- package/dist/cli.d.ts +0 -3
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -84
- package/dist/cypress/failure-explainer.d.ts +0 -45
- package/dist/cypress/failure-explainer.d.ts.map +0 -1
- package/dist/cypress/failure-explainer.js +0 -123
- package/dist/finder.d.ts +0 -68
- package/dist/finder.d.ts.map +0 -1
- package/dist/finder.js +0 -64
- package/dist/inventory.d.ts +0 -33
- package/dist/inventory.d.ts.map +0 -1
- package/dist/inventory.js +0 -88
- package/dist/measure.d.ts +0 -11
- package/dist/measure.d.ts.map +0 -1
- package/dist/measure.js +0 -145
- package/dist/pipeline.d.ts +0 -60
- package/dist/pipeline.d.ts.map +0 -1
- package/dist/pipeline.js +0 -115
- package/skills/annotate-bug/SKILL.md +0 -121
- package/src/browser/collect-elements.js +0 -158
- package/src/browser/measure-target.js +0 -69
package/dist/inventory.js
DELETED
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
import { readFile } from 'node:fs/promises';
|
|
2
|
-
import { fileURLToPath } from 'node:url';
|
|
3
|
-
import { dirname, join } from 'node:path';
|
|
4
|
-
const REF_ATTR = 'data-annot-ref';
|
|
5
|
-
const FRAME_ATTR = 'data-annot-frame';
|
|
6
|
-
const COLLECTOR_PATH = join(dirname(fileURLToPath(import.meta.url)), 'browser', 'collect-elements.js');
|
|
7
|
-
let collectorSource;
|
|
8
|
-
async function loadCollector() {
|
|
9
|
-
collectorSource ??= await readFile(COLLECTOR_PATH, 'utf8');
|
|
10
|
-
return collectorSource;
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Run the collector in a frame.
|
|
14
|
-
*
|
|
15
|
-
* The source is inlined into a self-invoking expression rather than passed as a
|
|
16
|
-
* function: Playwright will not call a function supplied as a string with
|
|
17
|
-
* arguments, and passing the TypeScript function directly breaks because
|
|
18
|
-
* esbuild rewrites it to reference a `__name` helper that the page lacks.
|
|
19
|
-
*/
|
|
20
|
-
async function runCollector(frame, args) {
|
|
21
|
-
const source = await loadCollector();
|
|
22
|
-
return frame.evaluate(`(${source})(${JSON.stringify(args)})`);
|
|
23
|
-
}
|
|
24
|
-
/** Tag same-origin child frames so their contents can be addressed by a chain. */
|
|
25
|
-
async function tagFrames(page) {
|
|
26
|
-
const map = new Map();
|
|
27
|
-
let index = 0;
|
|
28
|
-
for (const frame of page.frames()) {
|
|
29
|
-
if (frame === page.mainFrame())
|
|
30
|
-
continue;
|
|
31
|
-
try {
|
|
32
|
-
const element = await frame.frameElement();
|
|
33
|
-
const id = `f${++index}`;
|
|
34
|
-
await element.evaluate((el, [attr, value]) => el.setAttribute(attr, value), [FRAME_ATTR, id]);
|
|
35
|
-
map.set(frame, `[${FRAME_ATTR}="${id}"]`);
|
|
36
|
-
}
|
|
37
|
-
catch {
|
|
38
|
-
// Cross-origin or detached frames cannot be tagged; they are skipped and
|
|
39
|
-
// their contents simply will not appear in the inventory.
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
return map;
|
|
43
|
-
}
|
|
44
|
-
export async function collectInventory(page, options = {}) {
|
|
45
|
-
const limit = options.limit ?? 120;
|
|
46
|
-
const args = {
|
|
47
|
-
refAttr: REF_ATTR,
|
|
48
|
-
minSize: options.minSize ?? 8,
|
|
49
|
-
viewportOnly: options.viewportOnly ?? false,
|
|
50
|
-
};
|
|
51
|
-
const items = [];
|
|
52
|
-
const mainRaw = await runCollector(page.mainFrame(), { ...args, prefix: 'e' });
|
|
53
|
-
for (const raw of mainRaw) {
|
|
54
|
-
items.push({ ...raw, target: [`[${REF_ATTR}="${raw.ref}"]`], framePath: [] });
|
|
55
|
-
}
|
|
56
|
-
const frames = await tagFrames(page);
|
|
57
|
-
let frameIndex = 0;
|
|
58
|
-
for (const [frame, frameSelector] of frames) {
|
|
59
|
-
frameIndex++;
|
|
60
|
-
try {
|
|
61
|
-
const raw = await runCollector(frame, { ...args, prefix: `f${frameIndex}e` });
|
|
62
|
-
for (const item of raw) {
|
|
63
|
-
items.push({
|
|
64
|
-
...item,
|
|
65
|
-
target: [frameSelector, `[${REF_ATTR}="${item.ref}"]`],
|
|
66
|
-
framePath: [frameSelector],
|
|
67
|
-
});
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
catch {
|
|
71
|
-
// Same reasoning as tagFrames: an inaccessible frame is skipped.
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
return items.slice(0, limit);
|
|
75
|
-
}
|
|
76
|
-
/** Compact text rendering of the inventory for the model prompt. */
|
|
77
|
-
export function renderInventory(items) {
|
|
78
|
-
return items
|
|
79
|
-
.map((item) => {
|
|
80
|
-
const r = item.rect;
|
|
81
|
-
const box = `${Math.round(r.x)},${Math.round(r.y)} ${Math.round(r.width)}x${Math.round(r.height)}`;
|
|
82
|
-
const role = item.role ? ` role=${item.role}` : '';
|
|
83
|
-
const frame = item.framePath.length > 0 ? ' [in iframe]' : '';
|
|
84
|
-
const name = item.name ? ` "${item.name}"` : '';
|
|
85
|
-
return `[${item.ref}] <${item.tag}>${role}${name} @ ${box}${frame} — ${item.reportSelector}`;
|
|
86
|
-
})
|
|
87
|
-
.join('\n');
|
|
88
|
-
}
|
package/dist/measure.d.ts
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import type { Page } from 'playwright';
|
|
2
|
-
import type { MeasuredElement, PageMetrics, Region } from './types.js';
|
|
3
|
-
export declare function readPageMetrics(page: Page): Promise<PageMetrics>;
|
|
4
|
-
export declare function measureElement(page: Page, selector: string | string[], metrics: PageMetrics): Promise<MeasuredElement>;
|
|
5
|
-
/**
|
|
6
|
-
* Turn a percentage region into the same shape a measured element has, so the
|
|
7
|
-
* rest of the pipeline cannot tell the difference.
|
|
8
|
-
*/
|
|
9
|
-
export declare function measureRegion(region: Region, metrics: PageMetrics): MeasuredElement;
|
|
10
|
-
export declare function scrollElementIntoView(page: Page, selector: string | string[]): Promise<void>;
|
|
11
|
-
//# sourceMappingURL=measure.d.ts.map
|
package/dist/measure.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"measure.d.ts","sourceRoot":"","sources":["../src/measure.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAyB,MAAM,YAAY,CAAC;AAC9D,OAAO,KAAK,EAAW,eAAe,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAKhF,wBAAsB,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,CAoBtE;AAwCD,wBAAsB,cAAc,CAClC,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,EAC3B,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,eAAe,CAAC,CA+D1B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,eAAe,CAiCnF;AAED,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIlG"}
|
package/dist/measure.js
DELETED
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
export async function readPageMetrics(page) {
|
|
2
|
-
return page.evaluate(() => {
|
|
3
|
-
const doc = document.documentElement;
|
|
4
|
-
const body = document.body;
|
|
5
|
-
return {
|
|
6
|
-
devicePixelRatio: window.devicePixelRatio,
|
|
7
|
-
scrollX: window.scrollX,
|
|
8
|
-
scrollY: window.scrollY,
|
|
9
|
-
// window.innerWidth/Height, not documentElement.clientWidth/Height: in
|
|
10
|
-
// quirks mode (a page with no doctype - Hacker News, plenty of internal
|
|
11
|
-
// tools) the root element reports the *content* size, so clientHeight can
|
|
12
|
-
// be the full document height. innerWidth/Height is also exactly the
|
|
13
|
-
// viewport Playwright was configured with, which is what the screenshot
|
|
14
|
-
// dimensions are derived from.
|
|
15
|
-
viewportWidth: window.innerWidth,
|
|
16
|
-
viewportHeight: window.innerHeight,
|
|
17
|
-
documentWidth: Math.max(doc.scrollWidth, body ? body.scrollWidth : 0),
|
|
18
|
-
documentHeight: Math.max(doc.scrollHeight, body ? body.scrollHeight : 0),
|
|
19
|
-
};
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Walk a frame chain, accumulating the offset of each iframe's content origin
|
|
24
|
-
* within the top-level viewport. A frame's own internal scrolling needs no
|
|
25
|
-
* correction: `getBoundingClientRect()` inside that frame is already relative
|
|
26
|
-
* to the frame's own viewport.
|
|
27
|
-
*/
|
|
28
|
-
async function resolveTarget(page, chain) {
|
|
29
|
-
const targetSelector = chain[chain.length - 1];
|
|
30
|
-
if (targetSelector === undefined)
|
|
31
|
-
throw new Error('Empty selector chain.');
|
|
32
|
-
let host = page;
|
|
33
|
-
let offsetX = 0;
|
|
34
|
-
let offsetY = 0;
|
|
35
|
-
for (const frameSelector of chain.slice(0, -1)) {
|
|
36
|
-
const frameElement = host.locator(frameSelector);
|
|
37
|
-
if ((await frameElement.count()) === 0) {
|
|
38
|
-
throw new Error(`Frame selector matched nothing: ${frameSelector}`);
|
|
39
|
-
}
|
|
40
|
-
const origin = await frameElement.first().evaluate((el) => {
|
|
41
|
-
const rect = el.getBoundingClientRect();
|
|
42
|
-
const style = getComputedStyle(el);
|
|
43
|
-
return {
|
|
44
|
-
x: rect.left + parseFloat(style.borderLeftWidth) + parseFloat(style.paddingLeft),
|
|
45
|
-
y: rect.top + parseFloat(style.borderTopWidth) + parseFloat(style.paddingTop),
|
|
46
|
-
};
|
|
47
|
-
});
|
|
48
|
-
offsetX += origin.x;
|
|
49
|
-
offsetY += origin.y;
|
|
50
|
-
host = host.frameLocator(frameSelector);
|
|
51
|
-
}
|
|
52
|
-
return { locator: host.locator(targetSelector), offsetX, offsetY };
|
|
53
|
-
}
|
|
54
|
-
export async function measureElement(page, selector, metrics) {
|
|
55
|
-
const chain = Array.isArray(selector) ? selector : [selector];
|
|
56
|
-
const label = chain.join(' >>> ');
|
|
57
|
-
const { locator, offsetX, offsetY } = await resolveTarget(page, chain);
|
|
58
|
-
const matchCount = await locator.count();
|
|
59
|
-
if (matchCount === 0)
|
|
60
|
-
throw new Error(`Selector matched no elements: ${label}`);
|
|
61
|
-
const measured = await locator.first().evaluate((el) => {
|
|
62
|
-
const rect = el.getBoundingClientRect();
|
|
63
|
-
// An element inside a fixed subtree does not move with document scroll, so
|
|
64
|
-
// its document-space position is undefined in the usual sense.
|
|
65
|
-
let fixed = false;
|
|
66
|
-
for (let node = el; node; node = node.parentElement) {
|
|
67
|
-
if (getComputedStyle(node).position === 'fixed') {
|
|
68
|
-
fixed = true;
|
|
69
|
-
break;
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
return {
|
|
73
|
-
x: rect.x,
|
|
74
|
-
y: rect.y,
|
|
75
|
-
width: rect.width,
|
|
76
|
-
height: rect.height,
|
|
77
|
-
isFixed: fixed,
|
|
78
|
-
tagName: el.tagName.toLowerCase(),
|
|
79
|
-
};
|
|
80
|
-
});
|
|
81
|
-
const viewportRect = {
|
|
82
|
-
x: measured.x + offsetX,
|
|
83
|
-
y: measured.y + offsetY,
|
|
84
|
-
width: measured.width,
|
|
85
|
-
height: measured.height,
|
|
86
|
-
};
|
|
87
|
-
const documentRect = measured.isFixed
|
|
88
|
-
? { ...viewportRect }
|
|
89
|
-
: {
|
|
90
|
-
...viewportRect,
|
|
91
|
-
x: viewportRect.x + metrics.scrollX,
|
|
92
|
-
y: viewportRect.y + metrics.scrollY,
|
|
93
|
-
};
|
|
94
|
-
const inViewport = viewportRect.x + viewportRect.width > 0 &&
|
|
95
|
-
viewportRect.y + viewportRect.height > 0 &&
|
|
96
|
-
viewportRect.x < metrics.viewportWidth &&
|
|
97
|
-
viewportRect.y < metrics.viewportHeight;
|
|
98
|
-
return {
|
|
99
|
-
selector: label,
|
|
100
|
-
source: 'selector',
|
|
101
|
-
viewportRect,
|
|
102
|
-
documentRect,
|
|
103
|
-
isFixed: measured.isFixed,
|
|
104
|
-
inViewport,
|
|
105
|
-
framePath: chain.slice(0, -1),
|
|
106
|
-
tagName: measured.tagName,
|
|
107
|
-
matchCount,
|
|
108
|
-
};
|
|
109
|
-
}
|
|
110
|
-
/**
|
|
111
|
-
* Turn a percentage region into the same shape a measured element has, so the
|
|
112
|
-
* rest of the pipeline cannot tell the difference.
|
|
113
|
-
*/
|
|
114
|
-
export function measureRegion(region, metrics) {
|
|
115
|
-
const basis = region.basis ?? 'viewport';
|
|
116
|
-
const baseWidth = basis === 'document' ? metrics.documentWidth : metrics.viewportWidth;
|
|
117
|
-
const baseHeight = basis === 'document' ? metrics.documentHeight : metrics.viewportHeight;
|
|
118
|
-
const rect = {
|
|
119
|
-
x: (region.xPct / 100) * baseWidth,
|
|
120
|
-
y: (region.yPct / 100) * baseHeight,
|
|
121
|
-
width: (region.widthPct / 100) * baseWidth,
|
|
122
|
-
height: (region.heightPct / 100) * baseHeight,
|
|
123
|
-
};
|
|
124
|
-
const viewportRect = basis === 'document' ? { ...rect, x: rect.x - metrics.scrollX, y: rect.y - metrics.scrollY } : rect;
|
|
125
|
-
const documentRect = basis === 'document' ? rect : { ...rect, x: rect.x + metrics.scrollX, y: rect.y + metrics.scrollY };
|
|
126
|
-
return {
|
|
127
|
-
selector: `region ${region.xPct}%,${region.yPct}% ${region.widthPct}x${region.heightPct}% of ${basis}`,
|
|
128
|
-
source: 'region',
|
|
129
|
-
viewportRect,
|
|
130
|
-
documentRect,
|
|
131
|
-
isFixed: false,
|
|
132
|
-
inViewport: viewportRect.x + viewportRect.width > 0 &&
|
|
133
|
-
viewportRect.y + viewportRect.height > 0 &&
|
|
134
|
-
viewportRect.x < metrics.viewportWidth &&
|
|
135
|
-
viewportRect.y < metrics.viewportHeight,
|
|
136
|
-
framePath: [],
|
|
137
|
-
tagName: 'region',
|
|
138
|
-
matchCount: 1,
|
|
139
|
-
};
|
|
140
|
-
}
|
|
141
|
-
export async function scrollElementIntoView(page, selector) {
|
|
142
|
-
const chain = Array.isArray(selector) ? selector : [selector];
|
|
143
|
-
const { locator } = await resolveTarget(page, chain);
|
|
144
|
-
await locator.first().scrollIntoViewIfNeeded();
|
|
145
|
-
}
|
package/dist/pipeline.d.ts
DELETED
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import type { Page } from 'playwright';
|
|
2
|
-
import { type BugFinding, type Reasoner } from './finder.js';
|
|
3
|
-
import type { InventoryItem } from './inventory.js';
|
|
4
|
-
import type { Annotation, PixelRect } from './types.js';
|
|
5
|
-
export interface BugHuntOptions {
|
|
6
|
-
ticket: string;
|
|
7
|
-
reasoner: Reasoner;
|
|
8
|
-
/** Also render a cropped close-up per finding. Default true. */
|
|
9
|
-
perFinding?: boolean;
|
|
10
|
-
/** Context around each finding in its close-up, CSS px. Default 90. */
|
|
11
|
-
cropPadding?: number;
|
|
12
|
-
/** Cap on viewport passes down the page. Default 6. */
|
|
13
|
-
maxPasses?: number;
|
|
14
|
-
}
|
|
15
|
-
export interface AnnotatedFinding {
|
|
16
|
-
/** 1-based, stable across the whole hunt; matches the overview marker. */
|
|
17
|
-
number: number;
|
|
18
|
-
finding: BugFinding;
|
|
19
|
-
annotation: Annotation;
|
|
20
|
-
/** Which element the annotation actually landed on, for auditing. */
|
|
21
|
-
resolvedTo: string;
|
|
22
|
-
/** Readable selector for the ticket, when the finding resolved to an element. */
|
|
23
|
-
reportSelector: string | null;
|
|
24
|
-
/** Which viewport pass reported it. */
|
|
25
|
-
pass: number;
|
|
26
|
-
image: Buffer;
|
|
27
|
-
drawnRect: PixelRect | undefined;
|
|
28
|
-
warnings: string[];
|
|
29
|
-
}
|
|
30
|
-
export interface HuntPass {
|
|
31
|
-
index: number;
|
|
32
|
-
scrollY: number;
|
|
33
|
-
inventorySize: number;
|
|
34
|
-
findingCount: number;
|
|
35
|
-
/** Every finding from this pass on one viewport screenshot. */
|
|
36
|
-
overview: Buffer | null;
|
|
37
|
-
}
|
|
38
|
-
export interface BugHuntResult {
|
|
39
|
-
/** De-duplicated across passes. */
|
|
40
|
-
annotated: AnnotatedFinding[];
|
|
41
|
-
unresolved: {
|
|
42
|
-
finding: BugFinding;
|
|
43
|
-
reason: string;
|
|
44
|
-
}[];
|
|
45
|
-
passes: HuntPass[];
|
|
46
|
-
/** Union of every pass's inventory, keyed by readable selector. */
|
|
47
|
-
inventory: InventoryItem[];
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* Work down the page one viewport at a time, reasoning about each and
|
|
51
|
-
* annotating whatever comes back.
|
|
52
|
-
*
|
|
53
|
-
* A pass is the unit of correctness here: the inventory, the screenshot the
|
|
54
|
-
* model reasons over, and any percentage region it returns must all share one
|
|
55
|
-
* scroll position. Mixing scroll positions between those three is what makes
|
|
56
|
-
* region fallbacks land in the wrong place, so each pass scrolls, settles, and
|
|
57
|
-
* only then measures.
|
|
58
|
-
*/
|
|
59
|
-
export declare function runBugHunt(page: Page, options: BugHuntOptions): Promise<BugHuntResult>;
|
|
60
|
-
//# sourceMappingURL=pipeline.d.ts.map
|
package/dist/pipeline.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,QAAQ,EACd,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,gEAAgE;IAChE,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,iFAAiF;IACjF,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC;IACjC,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,UAAU,EAAE;QAAE,OAAO,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,mEAAmE;IACnE,SAAS,EAAE,aAAa,EAAE,CAAC;CAC5B;AAED;;;;;;;;;GASG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAqH5F"}
|
package/dist/pipeline.js
DELETED
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
import { annotate } from './annotate.js';
|
|
2
|
-
import { findingsToAnnotations, gatherReasonerInput, } from './finder.js';
|
|
3
|
-
import { readPageMetrics } from './measure.js';
|
|
4
|
-
/**
|
|
5
|
-
* Work down the page one viewport at a time, reasoning about each and
|
|
6
|
-
* annotating whatever comes back.
|
|
7
|
-
*
|
|
8
|
-
* A pass is the unit of correctness here: the inventory, the screenshot the
|
|
9
|
-
* model reasons over, and any percentage region it returns must all share one
|
|
10
|
-
* scroll position. Mixing scroll positions between those three is what makes
|
|
11
|
-
* region fallbacks land in the wrong place, so each pass scrolls, settles, and
|
|
12
|
-
* only then measures.
|
|
13
|
-
*/
|
|
14
|
-
export async function runBugHunt(page, options) {
|
|
15
|
-
const maxPasses = options.maxPasses ?? 6;
|
|
16
|
-
// Reuse the one definition of these metrics rather than re-deriving them:
|
|
17
|
-
// getting the viewport height wrong here silently collapses the sweep to a
|
|
18
|
-
// single pass on any quirks-mode page.
|
|
19
|
-
const { viewportHeight, documentHeight } = await readPageMetrics(page);
|
|
20
|
-
// Overlap each pass by 10% so a bug straddling a boundary is fully visible in
|
|
21
|
-
// at least one of them.
|
|
22
|
-
const step = Math.max(1, Math.floor(viewportHeight * 0.9));
|
|
23
|
-
const positions = [];
|
|
24
|
-
for (let y = 0; positions.length < maxPasses; y += step) {
|
|
25
|
-
const clamped = Math.min(y, Math.max(0, documentHeight - viewportHeight));
|
|
26
|
-
if (positions.length > 0 && clamped === positions[positions.length - 1])
|
|
27
|
-
break;
|
|
28
|
-
positions.push(clamped);
|
|
29
|
-
if (clamped + viewportHeight >= documentHeight)
|
|
30
|
-
break;
|
|
31
|
-
}
|
|
32
|
-
const annotated = [];
|
|
33
|
-
const unresolvedByDescription = new Map();
|
|
34
|
-
const passes = [];
|
|
35
|
-
const inventoryBySelector = new Map();
|
|
36
|
-
const seenKeys = new Set();
|
|
37
|
-
let findingNumber = 0;
|
|
38
|
-
for (const [passIndex, scrollY] of positions.entries()) {
|
|
39
|
-
await page.evaluate((y) => window.scrollTo(0, y), scrollY);
|
|
40
|
-
await page.waitForTimeout(120);
|
|
41
|
-
const input = await gatherReasonerInput(page, options.ticket, true);
|
|
42
|
-
for (const item of input.inventory) {
|
|
43
|
-
if (!inventoryBySelector.has(item.reportSelector)) {
|
|
44
|
-
inventoryBySelector.set(item.reportSelector, item);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
const findings = await options.reasoner.propose(input);
|
|
48
|
-
const resolution = findingsToAnnotations(findings, input.inventory);
|
|
49
|
-
for (const entry of resolution.unresolved) {
|
|
50
|
-
// An element below the fold on this pass may resolve on the next one, so
|
|
51
|
-
// key by description and reconcile against the annotated set at the end.
|
|
52
|
-
if (!unresolvedByDescription.has(entry.finding.description)) {
|
|
53
|
-
unresolvedByDescription.set(entry.finding.description, entry);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
// A bug visible in the overlap between two passes gets reported twice.
|
|
57
|
-
const fresh = resolution.resolved
|
|
58
|
-
.filter((entry) => !seenKeys.has(entry.key))
|
|
59
|
-
.map((entry) => ({ entry, number: ++findingNumber }));
|
|
60
|
-
for (const { entry } of fresh)
|
|
61
|
-
seenKeys.add(entry.key);
|
|
62
|
-
let overview = null;
|
|
63
|
-
if (fresh.length > 0) {
|
|
64
|
-
// Full sentences would cover the rest of the page here, so the overview
|
|
65
|
-
// carries a marker and the close-up carries the description.
|
|
66
|
-
const result = await annotate(page, fresh.map(({ entry, number }) => ({
|
|
67
|
-
...entry.annotation,
|
|
68
|
-
label: `#${number} ${entry.finding.severity}`,
|
|
69
|
-
})), { mode: 'viewport', scrollIntoView: false });
|
|
70
|
-
overview = result.image;
|
|
71
|
-
}
|
|
72
|
-
passes.push({
|
|
73
|
-
index: passIndex,
|
|
74
|
-
scrollY,
|
|
75
|
-
inventorySize: input.inventory.length,
|
|
76
|
-
findingCount: fresh.length,
|
|
77
|
-
overview,
|
|
78
|
-
});
|
|
79
|
-
if (options.perFinding ?? true) {
|
|
80
|
-
for (const { entry, number } of fresh) {
|
|
81
|
-
const labelled = {
|
|
82
|
-
...entry.annotation,
|
|
83
|
-
label: `#${number} ${entry.finding.severity}: ${entry.finding.description}`,
|
|
84
|
-
};
|
|
85
|
-
const result = await annotate(page, [labelled], {
|
|
86
|
-
mode: 'element',
|
|
87
|
-
cropPadding: options.cropPadding ?? 90,
|
|
88
|
-
// Staying at this pass's scroll position keeps percentage regions
|
|
89
|
-
// meaningful; annotate() converts to document space from here.
|
|
90
|
-
scrollToTop: false,
|
|
91
|
-
style: { dimOutside: 0.45 },
|
|
92
|
-
});
|
|
93
|
-
annotated.push({
|
|
94
|
-
number,
|
|
95
|
-
finding: entry.finding,
|
|
96
|
-
annotation: labelled,
|
|
97
|
-
resolvedTo: result.elements[0]?.selector ?? 'unknown',
|
|
98
|
-
reportSelector: entry.item?.reportSelector ?? null,
|
|
99
|
-
pass: passIndex,
|
|
100
|
-
image: result.image,
|
|
101
|
-
drawnRect: result.drawnRects[0],
|
|
102
|
-
warnings: result.warnings,
|
|
103
|
-
});
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
const annotatedDescriptions = new Set(annotated.map((a) => a.finding.description));
|
|
108
|
-
const unresolved = Array.from(unresolvedByDescription.values()).filter((entry) => !annotatedDescriptions.has(entry.finding.description));
|
|
109
|
-
return {
|
|
110
|
-
annotated,
|
|
111
|
-
unresolved,
|
|
112
|
-
passes,
|
|
113
|
-
inventory: Array.from(inventoryBySelector.values()),
|
|
114
|
-
};
|
|
115
|
-
}
|
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: annotate-bug
|
|
3
|
-
description: Draw an accurate box on a screenshot of the live page to show exactly where a visual bug is. Use after finding a layout or rendering defect during browser testing, when the bug report needs to point at a specific element rather than attach a raw screenshot. Works with whichever browser MCP backend is connected - Playwright MCP or Chrome DevTools MCP.
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
# Annotating a bug on a live page
|
|
7
|
-
|
|
8
|
-
Turns "the Apply button escapes its card" into a screenshot with a box drawn
|
|
9
|
-
around that exact element.
|
|
10
|
-
|
|
11
|
-
This skill does **not** open a browser. It annotates a screenshot that the
|
|
12
|
-
browser backend already connected to this session has captured. That matters:
|
|
13
|
-
the live session is the one that is logged in, and a separately launched browser
|
|
14
|
-
would land on the SSO login page instead of the page under test.
|
|
15
|
-
|
|
16
|
-
## Setup
|
|
17
|
-
|
|
18
|
-
Set `ANNOTATE_REPO` to wherever the tool is checked out (it needs
|
|
19
|
-
`npm install` once):
|
|
20
|
-
|
|
21
|
-
```
|
|
22
|
-
ANNOTATE_REPO=/path/to/cypress-annotate
|
|
23
|
-
```
|
|
24
|
-
|
|
25
|
-
## The protocol
|
|
26
|
-
|
|
27
|
-
Four steps, in this order. The ordering is the whole discipline: a rectangle is
|
|
28
|
-
only valid for the scroll position it was measured at.
|
|
29
|
-
|
|
30
|
-
### 1. Choose the element
|
|
31
|
-
|
|
32
|
-
Pick the single element that *is* wrong - not its container, and not the element
|
|
33
|
-
it collides with. If a page snapshot gave you an element ref, resolve it to a CSS
|
|
34
|
-
selector; prefer `#id` or `[data-testid=…]` over a structural path.
|
|
35
|
-
|
|
36
|
-
### 2. Measure it
|
|
37
|
-
|
|
38
|
-
Read `$ANNOTATE_REPO/src/browser/measure-target.js`, replace the `SELECTORS`
|
|
39
|
-
line with your selector(s), and evaluate the whole function in the page. Save the
|
|
40
|
-
result to a file.
|
|
41
|
-
|
|
42
|
-
**Playwright MCP:**
|
|
43
|
-
```
|
|
44
|
-
browser_evaluate({ function: "<the edited snippet>", filename: "/abs/path/measure.json" })
|
|
45
|
-
```
|
|
46
|
-
|
|
47
|
-
**Chrome DevTools MCP:**
|
|
48
|
-
```
|
|
49
|
-
evaluate_script({ pageId: <id>, function: "<the edited snippet>", filePath: "/abs/path/measure.json" })
|
|
50
|
-
```
|
|
51
|
-
|
|
52
|
-
Check the result: each target has `found: true` and a `rect`. If `found` is
|
|
53
|
-
false, the selector is wrong - fix it rather than proceeding, or fall back to a
|
|
54
|
-
region (below). If `matchCount` is above 1, the selector is ambiguous and the
|
|
55
|
-
first match gets annotated, which may not be the one you mean.
|
|
56
|
-
|
|
57
|
-
### 3. Screenshot immediately
|
|
58
|
-
|
|
59
|
-
Take the screenshot **without navigating, scrolling, clicking, or waiting for a
|
|
60
|
-
re-render in between**. Anything that moves the page invalidates the rectangles
|
|
61
|
-
from step 2 and the box will be drawn in the wrong place.
|
|
62
|
-
|
|
63
|
-
**Playwright MCP:**
|
|
64
|
-
```
|
|
65
|
-
browser_take_screenshot({ filename: "/abs/path/shot.png" })
|
|
66
|
-
```
|
|
67
|
-
|
|
68
|
-
**Chrome DevTools MCP:**
|
|
69
|
-
```
|
|
70
|
-
take_screenshot({ pageId: <id>, filePath: "/abs/path/shot.png", format: "png" })
|
|
71
|
-
```
|
|
72
|
-
|
|
73
|
-
Viewport capture is the default and is what you want. If you pass `fullPage`,
|
|
74
|
-
pass `--capture fullPage` in step 4 as well, or every box below the fold will be
|
|
75
|
-
misplaced.
|
|
76
|
-
|
|
77
|
-
### 4. Draw the box
|
|
78
|
-
|
|
79
|
-
```bash
|
|
80
|
-
cd $ANNOTATE_REPO && npm run annotate-shot -- \
|
|
81
|
-
--image /abs/path/shot.png \
|
|
82
|
-
--measurement /abs/path/measure.json \
|
|
83
|
-
--label "Apply button escapes its card" \
|
|
84
|
-
--out /abs/path/annotated.png
|
|
85
|
-
```
|
|
86
|
-
|
|
87
|
-
One `--label` per target, in the same order as `SELECTORS`. Useful extras:
|
|
88
|
-
|
|
89
|
-
- `--crop` - close-up of the element with the rest dimmed. Best for a ticket.
|
|
90
|
-
- `--capture fullPage` - required if the screenshot was a full-page capture.
|
|
91
|
-
- `--shape arrow` - point at it instead of boxing it.
|
|
92
|
-
- `--color '#FF6A00'` - e.g. colour by severity.
|
|
93
|
-
|
|
94
|
-
Then attach `annotated.png` to the report and say what is wrong in words as well.
|
|
95
|
-
|
|
96
|
-
## When there is no element to point at
|
|
97
|
-
|
|
98
|
-
If the defect is *absent* content - blank space where something should render -
|
|
99
|
-
there is no selector to measure. Skip step 2 and give a rectangle as percentages
|
|
100
|
-
of the viewport:
|
|
101
|
-
|
|
102
|
-
```bash
|
|
103
|
-
npm run annotate-shot -- --image shot.png --region 10,50,30,8 \
|
|
104
|
-
--label "Delivery estimate missing" --out annotated.png
|
|
105
|
-
```
|
|
106
|
-
|
|
107
|
-
`--region` is `x,y,width,height` in percent. Use it only as a fallback: a
|
|
108
|
-
selector is measured, a region is guessed.
|
|
109
|
-
|
|
110
|
-
## Checks worth reading
|
|
111
|
-
|
|
112
|
-
The command prints warnings. Two are worth acting on:
|
|
113
|
-
|
|
114
|
-
- *"matched N elements; annotated the first"* - the selector is ambiguous. Make
|
|
115
|
-
it specific and re-run.
|
|
116
|
-
- *"the image was probably rescaled"* - not a problem. Both backends can resize
|
|
117
|
-
screenshots, so the true scale is measured from the image rather than trusted
|
|
118
|
-
from `devicePixelRatio`. The box is still correct.
|
|
119
|
-
|
|
120
|
-
If the box lands in the wrong place, the cause is almost always that the page
|
|
121
|
-
moved between steps 2 and 3. Redo both, back to back.
|