cypress-annotate 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +40 -28
- package/CHANGELOG.md +142 -0
- package/DOCS/images/crop.png +0 -0
- package/DOCS/images/failure-hook.png +0 -0
- package/DOCS/internals.md +151 -0
- package/README.md +104 -329
- package/dist/annotate-image.d.ts +5 -0
- package/dist/annotate-image.d.ts.map +1 -1
- package/dist/annotate-image.js +16 -2
- package/dist/cypress/commands.d.ts +6 -1
- package/dist/cypress/commands.d.ts.map +1 -1
- package/dist/cypress/commands.js +13 -6
- package/dist/cypress/config.d.ts +42 -0
- package/dist/cypress/config.d.ts.map +1 -0
- package/dist/cypress/config.js +102 -0
- package/dist/cypress/failure-hook.d.ts.map +1 -1
- package/dist/cypress/failure-hook.js +10 -3
- package/dist/cypress/failure-report.d.ts +6 -0
- package/dist/cypress/failure-report.d.ts.map +1 -1
- package/dist/cypress/failure-report.js +14 -4
- package/dist/cypress/task.d.ts +36 -2
- package/dist/cypress/task.d.ts.map +1 -1
- package/dist/cypress/task.js +60 -5
- package/dist/draw.d.ts +4 -0
- package/dist/draw.d.ts.map +1 -1
- package/dist/draw.js +7 -4
- package/dist/index.d.ts +8 -28
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -37
- package/dist/types.d.ts +6 -72
- package/dist/types.d.ts.map +1 -1
- package/package.json +10 -39
- package/skills/cypress-annotate/SKILL.md +51 -9
- package/DOCS/pipeline.md +0 -308
- package/dist/annotate.d.ts +0 -9
- package/dist/annotate.d.ts.map +0 -1
- package/dist/annotate.js +0 -82
- package/dist/browser/collect-elements.js +0 -158
- package/dist/browser/measure-target.js +0 -69
- package/dist/claude-reasoner.d.ts +0 -17
- package/dist/claude-reasoner.d.ts.map +0 -1
- package/dist/claude-reasoner.js +0 -192
- package/dist/cli.d.ts +0 -2
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -74
- package/dist/cypress/failure-explainer.d.ts +0 -45
- package/dist/cypress/failure-explainer.d.ts.map +0 -1
- package/dist/cypress/failure-explainer.js +0 -123
- package/dist/finder.d.ts +0 -68
- package/dist/finder.d.ts.map +0 -1
- package/dist/finder.js +0 -64
- package/dist/inventory.d.ts +0 -33
- package/dist/inventory.d.ts.map +0 -1
- package/dist/inventory.js +0 -88
- package/dist/measure.d.ts +0 -11
- package/dist/measure.d.ts.map +0 -1
- package/dist/measure.js +0 -145
- package/dist/pipeline.d.ts +0 -60
- package/dist/pipeline.d.ts.map +0 -1
- package/dist/pipeline.js +0 -115
- package/skills/annotate-bug/SKILL.md +0 -121
- package/src/browser/collect-elements.js +0 -158
- package/src/browser/measure-target.js +0 -69
|
@@ -1,123 +0,0 @@
|
|
|
1
|
-
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
-
const SCHEMA = {
|
|
3
|
-
type: 'object',
|
|
4
|
-
properties: {
|
|
5
|
-
selector: {
|
|
6
|
-
type: ['string', 'null'],
|
|
7
|
-
description: 'The "selector" value of the single inventory element that best shows where this failure is ' +
|
|
8
|
-
'visible on screen. Null only if no listed element corresponds to it.',
|
|
9
|
-
},
|
|
10
|
-
label: {
|
|
11
|
-
type: 'string',
|
|
12
|
-
description: 'One short sentence for the annotation, explaining the failure in plain terms.',
|
|
13
|
-
},
|
|
14
|
-
region: {
|
|
15
|
-
type: ['object', 'null'],
|
|
16
|
-
description: 'Fallback used only when selector is null: the area to box, in percentages of the viewport.',
|
|
17
|
-
properties: {
|
|
18
|
-
xPct: { type: 'number' },
|
|
19
|
-
yPct: { type: 'number' },
|
|
20
|
-
widthPct: { type: 'number' },
|
|
21
|
-
heightPct: { type: 'number' },
|
|
22
|
-
},
|
|
23
|
-
required: ['xPct', 'yPct', 'widthPct', 'heightPct'],
|
|
24
|
-
additionalProperties: false,
|
|
25
|
-
},
|
|
26
|
-
},
|
|
27
|
-
required: ['selector', 'label', 'region'],
|
|
28
|
-
additionalProperties: false,
|
|
29
|
-
};
|
|
30
|
-
const SYSTEM_PROMPT = `A Cypress test failed. You are given the screenshot at the moment of failure,
|
|
31
|
-
the test's own error message, and a list of candidate elements on the page with their
|
|
32
|
-
tag, visible text, and bounding box in CSS pixels.
|
|
33
|
-
|
|
34
|
-
This is not exploratory bug hunting - the failure is already known and described by the
|
|
35
|
-
error message. Your only job is to say where on screen it is visible, by picking the
|
|
36
|
-
single best-matching element from the list, and to write one short, plain-language
|
|
37
|
-
sentence explaining it for someone triaging CI failures who has not read the error.
|
|
38
|
-
|
|
39
|
-
Prefer an element whose visible text or position plausibly explains the error message
|
|
40
|
-
over one that merely has a similar name. If nothing in the list corresponds to the
|
|
41
|
-
failure - for example the error describes something absent, like a modal that should
|
|
42
|
-
have appeared - set selector to null and give a region instead, as percentages of the
|
|
43
|
-
viewport, for roughly where it was expected.
|
|
44
|
-
|
|
45
|
-
Always call the report tool exactly once.`;
|
|
46
|
-
function renderInventory(inventory) {
|
|
47
|
-
if (!inventory)
|
|
48
|
-
return '(no candidate elements captured)';
|
|
49
|
-
return inventory
|
|
50
|
-
.map((item) => {
|
|
51
|
-
const r = item.rect;
|
|
52
|
-
const box = r ? `${Math.round(r.x)},${Math.round(r.y)} ${Math.round(r.width)}x${Math.round(r.height)}` : '?';
|
|
53
|
-
const text = item.text ? ` "${item.text}"` : '';
|
|
54
|
-
return `[${item.selector}] <${item.tag ?? '?'}>${text} @ ${box}`;
|
|
55
|
-
})
|
|
56
|
-
.join('\n');
|
|
57
|
-
}
|
|
58
|
-
export class ClaudeFailureExplainer {
|
|
59
|
-
client;
|
|
60
|
-
model;
|
|
61
|
-
effort;
|
|
62
|
-
constructor(options = {}) {
|
|
63
|
-
this.client = options.client ?? new Anthropic();
|
|
64
|
-
this.model = options.model ?? 'claude-opus-5';
|
|
65
|
-
this.effort = options.effort ?? 'medium';
|
|
66
|
-
}
|
|
67
|
-
async explain(record) {
|
|
68
|
-
const { readFile } = await import('node:fs/promises');
|
|
69
|
-
const screenshot = await readFile(record.screenshotPath);
|
|
70
|
-
const stream = this.client.messages.stream({
|
|
71
|
-
model: this.model,
|
|
72
|
-
max_tokens: 4096,
|
|
73
|
-
system: SYSTEM_PROMPT,
|
|
74
|
-
thinking: { type: 'adaptive' },
|
|
75
|
-
output_config: { effort: this.effort },
|
|
76
|
-
tools: [
|
|
77
|
-
{
|
|
78
|
-
name: 'report_location',
|
|
79
|
-
description: 'Report where the known failure is visible on screen.',
|
|
80
|
-
input_schema: SCHEMA,
|
|
81
|
-
strict: true,
|
|
82
|
-
},
|
|
83
|
-
],
|
|
84
|
-
messages: [
|
|
85
|
-
{
|
|
86
|
-
role: 'user',
|
|
87
|
-
content: [
|
|
88
|
-
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: screenshot.toString('base64') } },
|
|
89
|
-
{
|
|
90
|
-
type: 'text',
|
|
91
|
-
text: `Test: ${record.test}\n` +
|
|
92
|
-
`Error: ${record.errMessage}\n\n` +
|
|
93
|
-
`Candidate elements:\n${renderInventory(record.inventory)}`,
|
|
94
|
-
},
|
|
95
|
-
],
|
|
96
|
-
},
|
|
97
|
-
],
|
|
98
|
-
});
|
|
99
|
-
const message = await stream.finalMessage();
|
|
100
|
-
if (message.stop_reason === 'refusal') {
|
|
101
|
-
throw new Error(`Model declined (${message.stop_details?.category ?? 'unknown'}).`);
|
|
102
|
-
}
|
|
103
|
-
const call = message.content.find((b) => b.type === 'tool_use' && b.name === 'report_location');
|
|
104
|
-
if (!call)
|
|
105
|
-
throw new Error(`Model did not call report_location (stop_reason: ${message.stop_reason}).`);
|
|
106
|
-
const input = call.input;
|
|
107
|
-
return {
|
|
108
|
-
selector: typeof input.selector === 'string' ? input.selector : null,
|
|
109
|
-
label: typeof input.label === 'string' && input.label ? input.label : record.label,
|
|
110
|
-
region: input.region ?? null,
|
|
111
|
-
};
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
/** For tests and offline runs - no API call. */
|
|
115
|
-
export class ReplayFailureExplainer {
|
|
116
|
-
explanation;
|
|
117
|
-
constructor(explanation) {
|
|
118
|
-
this.explanation = explanation;
|
|
119
|
-
}
|
|
120
|
-
async explain() {
|
|
121
|
-
return this.explanation;
|
|
122
|
-
}
|
|
123
|
-
}
|
package/dist/finder.d.ts
DELETED
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
import type { Page } from 'playwright';
|
|
2
|
-
import { type InventoryItem } from './inventory.js';
|
|
3
|
-
import type { Annotation, Region } from './types.js';
|
|
4
|
-
export type Severity = 'blocker' | 'major' | 'minor' | 'cosmetic';
|
|
5
|
-
export interface BugFinding {
|
|
6
|
-
description: string;
|
|
7
|
-
/** Inventory handle, e.g. "e12". Null when the model could not pick one. */
|
|
8
|
-
ref: string | null;
|
|
9
|
-
/** Percentage fallback, used only when `ref` is null. */
|
|
10
|
-
region: Region | null;
|
|
11
|
-
severity: Severity;
|
|
12
|
-
/** What the model saw that made it call this a bug. */
|
|
13
|
-
evidence: string;
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* The reasoning step is an interface so the pipeline can be exercised with a
|
|
17
|
-
* recorded response, and so a different backend (computer use, an MCP server)
|
|
18
|
-
* can be dropped in without touching the annotation code.
|
|
19
|
-
*
|
|
20
|
-
* This file deliberately has no dependency on `@anthropic-ai/sdk` - see
|
|
21
|
-
* claude-reasoner.ts, which implements this interface and is the only file
|
|
22
|
-
* that does. Keeping them separate means importing anything here (from the
|
|
23
|
-
* package's root barrel) never forces the SDK to be installed.
|
|
24
|
-
*/
|
|
25
|
-
export interface Reasoner {
|
|
26
|
-
propose(input: ReasonerInput): Promise<BugFinding[]>;
|
|
27
|
-
}
|
|
28
|
-
export interface ReasonerInput {
|
|
29
|
-
screenshotPng: Buffer;
|
|
30
|
-
inventory: InventoryItem[];
|
|
31
|
-
ticket: string;
|
|
32
|
-
url: string;
|
|
33
|
-
}
|
|
34
|
-
/** A reasoner backed by a recorded response, for tests and offline runs. */
|
|
35
|
-
export declare class ReplayReasoner implements Reasoner {
|
|
36
|
-
private readonly findings;
|
|
37
|
-
constructor(findings: BugFinding[]);
|
|
38
|
-
propose(): Promise<BugFinding[]>;
|
|
39
|
-
}
|
|
40
|
-
export declare const SEVERITY_COLORS: Record<Severity, string>;
|
|
41
|
-
export interface ResolvedFinding {
|
|
42
|
-
finding: BugFinding;
|
|
43
|
-
annotation: Annotation;
|
|
44
|
-
/** Present when the finding resolved through an inventory handle. */
|
|
45
|
-
item?: InventoryItem;
|
|
46
|
-
/**
|
|
47
|
-
* Stable identity for de-duplicating the same bug seen in overlapping
|
|
48
|
-
* viewport passes. Inventory handles are re-issued on every pass, so this is
|
|
49
|
-
* keyed on the readable selector instead.
|
|
50
|
-
*/
|
|
51
|
-
key: string;
|
|
52
|
-
}
|
|
53
|
-
export interface ResolvedFindings {
|
|
54
|
-
resolved: ResolvedFinding[];
|
|
55
|
-
/** Findings that named neither a valid ref nor a usable region. */
|
|
56
|
-
unresolved: {
|
|
57
|
-
finding: BugFinding;
|
|
58
|
-
reason: string;
|
|
59
|
-
}[];
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Turn model findings into annotations. A ref is looked up in the inventory,
|
|
63
|
-
* which is what guarantees the selector resolves; a region is passed through;
|
|
64
|
-
* anything else is reported as unresolved rather than silently dropped.
|
|
65
|
-
*/
|
|
66
|
-
export declare function findingsToAnnotations(findings: BugFinding[], inventory: InventoryItem[]): ResolvedFindings;
|
|
67
|
-
export declare function gatherReasonerInput(page: Page, ticket: string, viewportOnly?: boolean): Promise<ReasonerInput>;
|
|
68
|
-
//# sourceMappingURL=finder.d.ts.map
|
package/dist/finder.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"finder.d.ts","sourceRoot":"","sources":["../src/finder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAoB,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAErD,MAAM,MAAM,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAC;AAElE,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,yDAAyD;IACzD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,QAAQ,CAAC;IACnB,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;CACb;AAED,4EAA4E;AAC5E,qBAAa,cAAe,YAAW,QAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,UAAU,EAAE;IAC7C,OAAO,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;CAGvC;AAED,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAKpD,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,qEAAqE;IACrE,IAAI,CAAC,EAAE,aAAa,CAAC;IACrB;;;;OAIG;IACH,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,mEAAmE;IACnE,UAAU,EAAE;QAAE,OAAO,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACvD;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,UAAU,EAAE,EACtB,SAAS,EAAE,aAAa,EAAE,GACzB,gBAAgB,CAyClB;AAED,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,MAAM,EACd,YAAY,UAAO,GAClB,OAAO,CAAC,aAAa,CAAC,CAIxB"}
|
package/dist/finder.js
DELETED
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
import { collectInventory } from './inventory.js';
|
|
2
|
-
/** A reasoner backed by a recorded response, for tests and offline runs. */
|
|
3
|
-
export class ReplayReasoner {
|
|
4
|
-
findings;
|
|
5
|
-
constructor(findings) {
|
|
6
|
-
this.findings = findings;
|
|
7
|
-
}
|
|
8
|
-
async propose() {
|
|
9
|
-
return this.findings;
|
|
10
|
-
}
|
|
11
|
-
}
|
|
12
|
-
export const SEVERITY_COLORS = {
|
|
13
|
-
blocker: '#FF2D2D',
|
|
14
|
-
major: '#FF6A00',
|
|
15
|
-
minor: '#F5A623',
|
|
16
|
-
cosmetic: '#3D7BFF',
|
|
17
|
-
};
|
|
18
|
-
/**
|
|
19
|
-
* Turn model findings into annotations. A ref is looked up in the inventory,
|
|
20
|
-
* which is what guarantees the selector resolves; a region is passed through;
|
|
21
|
-
* anything else is reported as unresolved rather than silently dropped.
|
|
22
|
-
*/
|
|
23
|
-
export function findingsToAnnotations(findings, inventory) {
|
|
24
|
-
const byRef = new Map(inventory.map((item) => [item.ref, item]));
|
|
25
|
-
const resolved = [];
|
|
26
|
-
const unresolved = [];
|
|
27
|
-
for (const finding of findings) {
|
|
28
|
-
const style = { color: SEVERITY_COLORS[finding.severity] };
|
|
29
|
-
const label = `${finding.severity}: ${finding.description}`;
|
|
30
|
-
if (finding.ref) {
|
|
31
|
-
const item = byRef.get(finding.ref);
|
|
32
|
-
if (item) {
|
|
33
|
-
resolved.push({
|
|
34
|
-
finding,
|
|
35
|
-
annotation: { selector: item.target, label, style },
|
|
36
|
-
item,
|
|
37
|
-
key: `sel:${item.reportSelector}`,
|
|
38
|
-
});
|
|
39
|
-
continue;
|
|
40
|
-
}
|
|
41
|
-
// A hallucinated handle is recoverable if a region came with it.
|
|
42
|
-
if (!finding.region) {
|
|
43
|
-
unresolved.push({ finding, reason: `ref "${finding.ref}" is not in the inventory` });
|
|
44
|
-
continue;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
if (finding.region) {
|
|
48
|
-
const r = finding.region;
|
|
49
|
-
resolved.push({
|
|
50
|
-
finding,
|
|
51
|
-
annotation: { region: r, label, style },
|
|
52
|
-
key: `region:${Math.round(r.xPct)},${Math.round(r.yPct)},${Math.round(r.widthPct)},${Math.round(r.heightPct)}`,
|
|
53
|
-
});
|
|
54
|
-
continue;
|
|
55
|
-
}
|
|
56
|
-
unresolved.push({ finding, reason: 'no ref and no region' });
|
|
57
|
-
}
|
|
58
|
-
return { resolved, unresolved };
|
|
59
|
-
}
|
|
60
|
-
export async function gatherReasonerInput(page, ticket, viewportOnly = true) {
|
|
61
|
-
const inventory = await collectInventory(page, { viewportOnly });
|
|
62
|
-
const screenshotPng = await page.screenshot({ animations: 'disabled', caret: 'hide', type: 'png' });
|
|
63
|
-
return { screenshotPng, inventory, ticket, url: page.url() };
|
|
64
|
-
}
|
package/dist/inventory.d.ts
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
import type { Page } from 'playwright';
|
|
2
|
-
import type { CssRect } from './types.js';
|
|
3
|
-
export interface InventoryItem {
|
|
4
|
-
/** Handle the model refers to, e.g. "e12". */
|
|
5
|
-
ref: string;
|
|
6
|
-
/**
|
|
7
|
-
* Selector chain that is guaranteed to resolve, because the attribute it
|
|
8
|
-
* matches was injected during collection.
|
|
9
|
-
*/
|
|
10
|
-
target: string[];
|
|
11
|
-
/** Readable selector for the bug report, derived from ids/test-ids/structure. */
|
|
12
|
-
reportSelector: string;
|
|
13
|
-
tag: string;
|
|
14
|
-
role: string | null;
|
|
15
|
-
/** Accessible-ish name: aria-label, alt, placeholder, value, or trimmed text. */
|
|
16
|
-
name: string;
|
|
17
|
-
rect: CssRect;
|
|
18
|
-
inViewport: boolean;
|
|
19
|
-
/** Empty for the main document. */
|
|
20
|
-
framePath: string[];
|
|
21
|
-
}
|
|
22
|
-
export interface InventoryOptions {
|
|
23
|
-
/** Cap on items sent to the model, to bound token cost. Default 120. */
|
|
24
|
-
limit?: number;
|
|
25
|
-
/** Ignore anything smaller than this many CSS px on either axis. Default 8. */
|
|
26
|
-
minSize?: number;
|
|
27
|
-
/** Only collect elements currently intersecting the viewport. Default false. */
|
|
28
|
-
viewportOnly?: boolean;
|
|
29
|
-
}
|
|
30
|
-
export declare function collectInventory(page: Page, options?: InventoryOptions): Promise<InventoryItem[]>;
|
|
31
|
-
/** Compact text rendering of the inventory for the model prompt. */
|
|
32
|
-
export declare function renderInventory(items: InventoryItem[]): string;
|
|
33
|
-
//# sourceMappingURL=inventory.d.ts.map
|
package/dist/inventory.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"inventory.d.ts","sourceRoot":"","sources":["../src/inventory.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAS,IAAI,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,WAAW,aAAa;IAC5B,8CAA8C;IAC9C,GAAG,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,iFAAiF;IACjF,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,OAAO,CAAC;IACpB,mCAAmC;IACnC,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,wEAAwE;IACxE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gFAAgF;IAChF,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AA8DD,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,IAAI,EACV,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,aAAa,EAAE,CAAC,CAkC1B;AAED,oEAAoE;AACpE,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,MAAM,CAW9D"}
|
package/dist/inventory.js
DELETED
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
import { readFile } from 'node:fs/promises';
|
|
2
|
-
import { fileURLToPath } from 'node:url';
|
|
3
|
-
import { dirname, join } from 'node:path';
|
|
4
|
-
const REF_ATTR = 'data-annot-ref';
|
|
5
|
-
const FRAME_ATTR = 'data-annot-frame';
|
|
6
|
-
const COLLECTOR_PATH = join(dirname(fileURLToPath(import.meta.url)), 'browser', 'collect-elements.js');
|
|
7
|
-
let collectorSource;
|
|
8
|
-
async function loadCollector() {
|
|
9
|
-
collectorSource ??= await readFile(COLLECTOR_PATH, 'utf8');
|
|
10
|
-
return collectorSource;
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Run the collector in a frame.
|
|
14
|
-
*
|
|
15
|
-
* The source is inlined into a self-invoking expression rather than passed as a
|
|
16
|
-
* function: Playwright will not call a function supplied as a string with
|
|
17
|
-
* arguments, and passing the TypeScript function directly breaks because
|
|
18
|
-
* esbuild rewrites it to reference a `__name` helper that the page lacks.
|
|
19
|
-
*/
|
|
20
|
-
async function runCollector(frame, args) {
|
|
21
|
-
const source = await loadCollector();
|
|
22
|
-
return frame.evaluate(`(${source})(${JSON.stringify(args)})`);
|
|
23
|
-
}
|
|
24
|
-
/** Tag same-origin child frames so their contents can be addressed by a chain. */
|
|
25
|
-
async function tagFrames(page) {
|
|
26
|
-
const map = new Map();
|
|
27
|
-
let index = 0;
|
|
28
|
-
for (const frame of page.frames()) {
|
|
29
|
-
if (frame === page.mainFrame())
|
|
30
|
-
continue;
|
|
31
|
-
try {
|
|
32
|
-
const element = await frame.frameElement();
|
|
33
|
-
const id = `f${++index}`;
|
|
34
|
-
await element.evaluate((el, [attr, value]) => el.setAttribute(attr, value), [FRAME_ATTR, id]);
|
|
35
|
-
map.set(frame, `[${FRAME_ATTR}="${id}"]`);
|
|
36
|
-
}
|
|
37
|
-
catch {
|
|
38
|
-
// Cross-origin or detached frames cannot be tagged; they are skipped and
|
|
39
|
-
// their contents simply will not appear in the inventory.
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
return map;
|
|
43
|
-
}
|
|
44
|
-
export async function collectInventory(page, options = {}) {
|
|
45
|
-
const limit = options.limit ?? 120;
|
|
46
|
-
const args = {
|
|
47
|
-
refAttr: REF_ATTR,
|
|
48
|
-
minSize: options.minSize ?? 8,
|
|
49
|
-
viewportOnly: options.viewportOnly ?? false,
|
|
50
|
-
};
|
|
51
|
-
const items = [];
|
|
52
|
-
const mainRaw = await runCollector(page.mainFrame(), { ...args, prefix: 'e' });
|
|
53
|
-
for (const raw of mainRaw) {
|
|
54
|
-
items.push({ ...raw, target: [`[${REF_ATTR}="${raw.ref}"]`], framePath: [] });
|
|
55
|
-
}
|
|
56
|
-
const frames = await tagFrames(page);
|
|
57
|
-
let frameIndex = 0;
|
|
58
|
-
for (const [frame, frameSelector] of frames) {
|
|
59
|
-
frameIndex++;
|
|
60
|
-
try {
|
|
61
|
-
const raw = await runCollector(frame, { ...args, prefix: `f${frameIndex}e` });
|
|
62
|
-
for (const item of raw) {
|
|
63
|
-
items.push({
|
|
64
|
-
...item,
|
|
65
|
-
target: [frameSelector, `[${REF_ATTR}="${item.ref}"]`],
|
|
66
|
-
framePath: [frameSelector],
|
|
67
|
-
});
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
catch {
|
|
71
|
-
// Same reasoning as tagFrames: an inaccessible frame is skipped.
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
return items.slice(0, limit);
|
|
75
|
-
}
|
|
76
|
-
/** Compact text rendering of the inventory for the model prompt. */
|
|
77
|
-
export function renderInventory(items) {
|
|
78
|
-
return items
|
|
79
|
-
.map((item) => {
|
|
80
|
-
const r = item.rect;
|
|
81
|
-
const box = `${Math.round(r.x)},${Math.round(r.y)} ${Math.round(r.width)}x${Math.round(r.height)}`;
|
|
82
|
-
const role = item.role ? ` role=${item.role}` : '';
|
|
83
|
-
const frame = item.framePath.length > 0 ? ' [in iframe]' : '';
|
|
84
|
-
const name = item.name ? ` "${item.name}"` : '';
|
|
85
|
-
return `[${item.ref}] <${item.tag}>${role}${name} @ ${box}${frame} — ${item.reportSelector}`;
|
|
86
|
-
})
|
|
87
|
-
.join('\n');
|
|
88
|
-
}
|
package/dist/measure.d.ts
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import type { Page } from 'playwright';
|
|
2
|
-
import type { MeasuredElement, PageMetrics, Region } from './types.js';
|
|
3
|
-
export declare function readPageMetrics(page: Page): Promise<PageMetrics>;
|
|
4
|
-
export declare function measureElement(page: Page, selector: string | string[], metrics: PageMetrics): Promise<MeasuredElement>;
|
|
5
|
-
/**
|
|
6
|
-
* Turn a percentage region into the same shape a measured element has, so the
|
|
7
|
-
* rest of the pipeline cannot tell the difference.
|
|
8
|
-
*/
|
|
9
|
-
export declare function measureRegion(region: Region, metrics: PageMetrics): MeasuredElement;
|
|
10
|
-
export declare function scrollElementIntoView(page: Page, selector: string | string[]): Promise<void>;
|
|
11
|
-
//# sourceMappingURL=measure.d.ts.map
|
package/dist/measure.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"measure.d.ts","sourceRoot":"","sources":["../src/measure.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAyB,MAAM,YAAY,CAAC;AAC9D,OAAO,KAAK,EAAW,eAAe,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAKhF,wBAAsB,eAAe,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,CAoBtE;AAwCD,wBAAsB,cAAc,CAClC,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,EAC3B,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,eAAe,CAAC,CA+D1B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,eAAe,CAiCnF;AAED,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAIlG"}
|
package/dist/measure.js
DELETED
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
export async function readPageMetrics(page) {
|
|
2
|
-
return page.evaluate(() => {
|
|
3
|
-
const doc = document.documentElement;
|
|
4
|
-
const body = document.body;
|
|
5
|
-
return {
|
|
6
|
-
devicePixelRatio: window.devicePixelRatio,
|
|
7
|
-
scrollX: window.scrollX,
|
|
8
|
-
scrollY: window.scrollY,
|
|
9
|
-
// window.innerWidth/Height, not documentElement.clientWidth/Height: in
|
|
10
|
-
// quirks mode (a page with no doctype - Hacker News, plenty of internal
|
|
11
|
-
// tools) the root element reports the *content* size, so clientHeight can
|
|
12
|
-
// be the full document height. innerWidth/Height is also exactly the
|
|
13
|
-
// viewport Playwright was configured with, which is what the screenshot
|
|
14
|
-
// dimensions are derived from.
|
|
15
|
-
viewportWidth: window.innerWidth,
|
|
16
|
-
viewportHeight: window.innerHeight,
|
|
17
|
-
documentWidth: Math.max(doc.scrollWidth, body ? body.scrollWidth : 0),
|
|
18
|
-
documentHeight: Math.max(doc.scrollHeight, body ? body.scrollHeight : 0),
|
|
19
|
-
};
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Walk a frame chain, accumulating the offset of each iframe's content origin
|
|
24
|
-
* within the top-level viewport. A frame's own internal scrolling needs no
|
|
25
|
-
* correction: `getBoundingClientRect()` inside that frame is already relative
|
|
26
|
-
* to the frame's own viewport.
|
|
27
|
-
*/
|
|
28
|
-
async function resolveTarget(page, chain) {
|
|
29
|
-
const targetSelector = chain[chain.length - 1];
|
|
30
|
-
if (targetSelector === undefined)
|
|
31
|
-
throw new Error('Empty selector chain.');
|
|
32
|
-
let host = page;
|
|
33
|
-
let offsetX = 0;
|
|
34
|
-
let offsetY = 0;
|
|
35
|
-
for (const frameSelector of chain.slice(0, -1)) {
|
|
36
|
-
const frameElement = host.locator(frameSelector);
|
|
37
|
-
if ((await frameElement.count()) === 0) {
|
|
38
|
-
throw new Error(`Frame selector matched nothing: ${frameSelector}`);
|
|
39
|
-
}
|
|
40
|
-
const origin = await frameElement.first().evaluate((el) => {
|
|
41
|
-
const rect = el.getBoundingClientRect();
|
|
42
|
-
const style = getComputedStyle(el);
|
|
43
|
-
return {
|
|
44
|
-
x: rect.left + parseFloat(style.borderLeftWidth) + parseFloat(style.paddingLeft),
|
|
45
|
-
y: rect.top + parseFloat(style.borderTopWidth) + parseFloat(style.paddingTop),
|
|
46
|
-
};
|
|
47
|
-
});
|
|
48
|
-
offsetX += origin.x;
|
|
49
|
-
offsetY += origin.y;
|
|
50
|
-
host = host.frameLocator(frameSelector);
|
|
51
|
-
}
|
|
52
|
-
return { locator: host.locator(targetSelector), offsetX, offsetY };
|
|
53
|
-
}
|
|
54
|
-
export async function measureElement(page, selector, metrics) {
|
|
55
|
-
const chain = Array.isArray(selector) ? selector : [selector];
|
|
56
|
-
const label = chain.join(' >>> ');
|
|
57
|
-
const { locator, offsetX, offsetY } = await resolveTarget(page, chain);
|
|
58
|
-
const matchCount = await locator.count();
|
|
59
|
-
if (matchCount === 0)
|
|
60
|
-
throw new Error(`Selector matched no elements: ${label}`);
|
|
61
|
-
const measured = await locator.first().evaluate((el) => {
|
|
62
|
-
const rect = el.getBoundingClientRect();
|
|
63
|
-
// An element inside a fixed subtree does not move with document scroll, so
|
|
64
|
-
// its document-space position is undefined in the usual sense.
|
|
65
|
-
let fixed = false;
|
|
66
|
-
for (let node = el; node; node = node.parentElement) {
|
|
67
|
-
if (getComputedStyle(node).position === 'fixed') {
|
|
68
|
-
fixed = true;
|
|
69
|
-
break;
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
return {
|
|
73
|
-
x: rect.x,
|
|
74
|
-
y: rect.y,
|
|
75
|
-
width: rect.width,
|
|
76
|
-
height: rect.height,
|
|
77
|
-
isFixed: fixed,
|
|
78
|
-
tagName: el.tagName.toLowerCase(),
|
|
79
|
-
};
|
|
80
|
-
});
|
|
81
|
-
const viewportRect = {
|
|
82
|
-
x: measured.x + offsetX,
|
|
83
|
-
y: measured.y + offsetY,
|
|
84
|
-
width: measured.width,
|
|
85
|
-
height: measured.height,
|
|
86
|
-
};
|
|
87
|
-
const documentRect = measured.isFixed
|
|
88
|
-
? { ...viewportRect }
|
|
89
|
-
: {
|
|
90
|
-
...viewportRect,
|
|
91
|
-
x: viewportRect.x + metrics.scrollX,
|
|
92
|
-
y: viewportRect.y + metrics.scrollY,
|
|
93
|
-
};
|
|
94
|
-
const inViewport = viewportRect.x + viewportRect.width > 0 &&
|
|
95
|
-
viewportRect.y + viewportRect.height > 0 &&
|
|
96
|
-
viewportRect.x < metrics.viewportWidth &&
|
|
97
|
-
viewportRect.y < metrics.viewportHeight;
|
|
98
|
-
return {
|
|
99
|
-
selector: label,
|
|
100
|
-
source: 'selector',
|
|
101
|
-
viewportRect,
|
|
102
|
-
documentRect,
|
|
103
|
-
isFixed: measured.isFixed,
|
|
104
|
-
inViewport,
|
|
105
|
-
framePath: chain.slice(0, -1),
|
|
106
|
-
tagName: measured.tagName,
|
|
107
|
-
matchCount,
|
|
108
|
-
};
|
|
109
|
-
}
|
|
110
|
-
/**
|
|
111
|
-
* Turn a percentage region into the same shape a measured element has, so the
|
|
112
|
-
* rest of the pipeline cannot tell the difference.
|
|
113
|
-
*/
|
|
114
|
-
export function measureRegion(region, metrics) {
|
|
115
|
-
const basis = region.basis ?? 'viewport';
|
|
116
|
-
const baseWidth = basis === 'document' ? metrics.documentWidth : metrics.viewportWidth;
|
|
117
|
-
const baseHeight = basis === 'document' ? metrics.documentHeight : metrics.viewportHeight;
|
|
118
|
-
const rect = {
|
|
119
|
-
x: (region.xPct / 100) * baseWidth,
|
|
120
|
-
y: (region.yPct / 100) * baseHeight,
|
|
121
|
-
width: (region.widthPct / 100) * baseWidth,
|
|
122
|
-
height: (region.heightPct / 100) * baseHeight,
|
|
123
|
-
};
|
|
124
|
-
const viewportRect = basis === 'document' ? { ...rect, x: rect.x - metrics.scrollX, y: rect.y - metrics.scrollY } : rect;
|
|
125
|
-
const documentRect = basis === 'document' ? rect : { ...rect, x: rect.x + metrics.scrollX, y: rect.y + metrics.scrollY };
|
|
126
|
-
return {
|
|
127
|
-
selector: `region ${region.xPct}%,${region.yPct}% ${region.widthPct}x${region.heightPct}% of ${basis}`,
|
|
128
|
-
source: 'region',
|
|
129
|
-
viewportRect,
|
|
130
|
-
documentRect,
|
|
131
|
-
isFixed: false,
|
|
132
|
-
inViewport: viewportRect.x + viewportRect.width > 0 &&
|
|
133
|
-
viewportRect.y + viewportRect.height > 0 &&
|
|
134
|
-
viewportRect.x < metrics.viewportWidth &&
|
|
135
|
-
viewportRect.y < metrics.viewportHeight,
|
|
136
|
-
framePath: [],
|
|
137
|
-
tagName: 'region',
|
|
138
|
-
matchCount: 1,
|
|
139
|
-
};
|
|
140
|
-
}
|
|
141
|
-
export async function scrollElementIntoView(page, selector) {
|
|
142
|
-
const chain = Array.isArray(selector) ? selector : [selector];
|
|
143
|
-
const { locator } = await resolveTarget(page, chain);
|
|
144
|
-
await locator.first().scrollIntoViewIfNeeded();
|
|
145
|
-
}
|
package/dist/pipeline.d.ts
DELETED
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import type { Page } from 'playwright';
|
|
2
|
-
import { type BugFinding, type Reasoner } from './finder.js';
|
|
3
|
-
import type { InventoryItem } from './inventory.js';
|
|
4
|
-
import type { Annotation, PixelRect } from './types.js';
|
|
5
|
-
export interface BugHuntOptions {
|
|
6
|
-
ticket: string;
|
|
7
|
-
reasoner: Reasoner;
|
|
8
|
-
/** Also render a cropped close-up per finding. Default true. */
|
|
9
|
-
perFinding?: boolean;
|
|
10
|
-
/** Context around each finding in its close-up, CSS px. Default 90. */
|
|
11
|
-
cropPadding?: number;
|
|
12
|
-
/** Cap on viewport passes down the page. Default 6. */
|
|
13
|
-
maxPasses?: number;
|
|
14
|
-
}
|
|
15
|
-
export interface AnnotatedFinding {
|
|
16
|
-
/** 1-based, stable across the whole hunt; matches the overview marker. */
|
|
17
|
-
number: number;
|
|
18
|
-
finding: BugFinding;
|
|
19
|
-
annotation: Annotation;
|
|
20
|
-
/** Which element the annotation actually landed on, for auditing. */
|
|
21
|
-
resolvedTo: string;
|
|
22
|
-
/** Readable selector for the ticket, when the finding resolved to an element. */
|
|
23
|
-
reportSelector: string | null;
|
|
24
|
-
/** Which viewport pass reported it. */
|
|
25
|
-
pass: number;
|
|
26
|
-
image: Buffer;
|
|
27
|
-
drawnRect: PixelRect | undefined;
|
|
28
|
-
warnings: string[];
|
|
29
|
-
}
|
|
30
|
-
export interface HuntPass {
|
|
31
|
-
index: number;
|
|
32
|
-
scrollY: number;
|
|
33
|
-
inventorySize: number;
|
|
34
|
-
findingCount: number;
|
|
35
|
-
/** Every finding from this pass on one viewport screenshot. */
|
|
36
|
-
overview: Buffer | null;
|
|
37
|
-
}
|
|
38
|
-
export interface BugHuntResult {
|
|
39
|
-
/** De-duplicated across passes. */
|
|
40
|
-
annotated: AnnotatedFinding[];
|
|
41
|
-
unresolved: {
|
|
42
|
-
finding: BugFinding;
|
|
43
|
-
reason: string;
|
|
44
|
-
}[];
|
|
45
|
-
passes: HuntPass[];
|
|
46
|
-
/** Union of every pass's inventory, keyed by readable selector. */
|
|
47
|
-
inventory: InventoryItem[];
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* Work down the page one viewport at a time, reasoning about each and
|
|
51
|
-
* annotating whatever comes back.
|
|
52
|
-
*
|
|
53
|
-
* A pass is the unit of correctness here: the inventory, the screenshot the
|
|
54
|
-
* model reasons over, and any percentage region it returns must all share one
|
|
55
|
-
* scroll position. Mixing scroll positions between those three is what makes
|
|
56
|
-
* region fallbacks land in the wrong place, so each pass scrolls, settles, and
|
|
57
|
-
* only then measures.
|
|
58
|
-
*/
|
|
59
|
-
export declare function runBugHunt(page: Page, options: BugHuntOptions): Promise<BugHuntResult>;
|
|
60
|
-
//# sourceMappingURL=pipeline.d.ts.map
|
package/dist/pipeline.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../src/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAGL,KAAK,UAAU,EACf,KAAK,QAAQ,EACd,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAExD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,gEAAgE;IAChE,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,iFAAiF;IACjF,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC;IACjC,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,+DAA+D;IAC/D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,UAAU,EAAE;QAAE,OAAO,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,mEAAmE;IACnE,SAAS,EAAE,aAAa,EAAE,CAAC;CAC5B;AAED;;;;;;;;;GASG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAqH5F"}
|