cypress-annotate 0.4.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +40 -28
- package/CHANGELOG.md +174 -0
- package/DOCS/images/crop.png +0 -0
- package/DOCS/images/failure-hook.png +0 -0
- package/DOCS/internals.md +151 -0
- package/README.md +104 -352
- package/dist/annotate-image.d.ts +19 -2
- package/dist/annotate-image.d.ts.map +1 -1
- package/dist/annotate-image.js +55 -12
- package/dist/cypress/commands.d.ts +6 -1
- package/dist/cypress/commands.d.ts.map +1 -1
- package/dist/cypress/commands.js +20 -6
- package/dist/cypress/config.d.ts +42 -0
- package/dist/cypress/config.d.ts.map +1 -0
- package/dist/cypress/config.js +102 -0
- package/dist/cypress/failure-hook.d.ts.map +1 -1
- package/dist/cypress/failure-hook.js +17 -3
- package/dist/cypress/failure-report.d.ts +6 -0
- package/dist/cypress/failure-report.d.ts.map +1 -1
- package/dist/cypress/failure-report.js +14 -4
- package/dist/cypress/task.d.ts +39 -3
- package/dist/cypress/task.d.ts.map +1 -1
- package/dist/cypress/task.js +61 -5
- package/dist/draw.d.ts +4 -0
- package/dist/draw.d.ts.map +1 -1
- package/dist/draw.js +7 -4
- package/dist/index.d.ts +9 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -57
- package/dist/types.d.ts +13 -69
- package/dist/types.d.ts.map +1 -1
- package/package.json +11 -43
- package/skills/cypress-annotate/SKILL.md +62 -9
- package/DOCS/pipeline.md +0 -308
- package/dist/annotate.d.ts +0 -9
- package/dist/annotate.d.ts.map +0 -1
- package/dist/annotate.js +0 -82
- package/dist/browser/collect-elements.js +0 -158
- package/dist/browser/measure-target.js +0 -69
- package/dist/claude-reasoner.d.ts +0 -17
- package/dist/claude-reasoner.d.ts.map +0 -1
- package/dist/claude-reasoner.js +0 -192
- package/dist/cli.d.ts +0 -3
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -84
- package/dist/cypress/failure-explainer.d.ts +0 -45
- package/dist/cypress/failure-explainer.d.ts.map +0 -1
- package/dist/cypress/failure-explainer.js +0 -123
- package/dist/finder.d.ts +0 -68
- package/dist/finder.d.ts.map +0 -1
- package/dist/finder.js +0 -64
- package/dist/inventory.d.ts +0 -33
- package/dist/inventory.d.ts.map +0 -1
- package/dist/inventory.js +0 -88
- package/dist/measure.d.ts +0 -11
- package/dist/measure.d.ts.map +0 -1
- package/dist/measure.js +0 -145
- package/dist/pipeline.d.ts +0 -60
- package/dist/pipeline.d.ts.map +0 -1
- package/dist/pipeline.js +0 -115
- package/skills/annotate-bug/SKILL.md +0 -121
- package/src/browser/collect-elements.js +0 -158
- package/src/browser/measure-target.js +0 -69
package/dist/claude-reasoner.js
DELETED
|
@@ -1,192 +0,0 @@
|
|
|
1
|
-
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
-
import { renderInventory } from './inventory.js';
|
|
3
|
-
/**
|
|
4
|
-
* Split out from finder.ts on purpose: this is the only file in the package
|
|
5
|
-
* with a real (non-type-only) import of `@anthropic-ai/sdk`. finder.ts's other
|
|
6
|
-
* exports (findingsToAnnotations, gatherReasonerInput, ReplayReasoner, ...) do
|
|
7
|
-
* not need the SDK at all, and index.ts re-exports them from its root barrel -
|
|
8
|
-
* if ClaudeReasoner lived in the same file, importing anything from that
|
|
9
|
-
* barrel would eagerly load the SDK too, since ES module imports evaluate the
|
|
10
|
-
* whole file regardless of which export you asked for. Caught by actually
|
|
11
|
-
* installing the built package into a scratch project and importing it with
|
|
12
|
-
* no SDK present, not by inspection.
|
|
13
|
-
*/
|
|
14
|
-
const SEVERITIES = ['blocker', 'major', 'minor', 'cosmetic'];
|
|
15
|
-
const FINDING_SCHEMA = {
|
|
16
|
-
type: 'object',
|
|
17
|
-
properties: {
|
|
18
|
-
findings: {
|
|
19
|
-
type: 'array',
|
|
20
|
-
description: 'Every visual or functional defect found. Empty if the page looks correct.',
|
|
21
|
-
items: {
|
|
22
|
-
type: 'object',
|
|
23
|
-
properties: {
|
|
24
|
-
description: {
|
|
25
|
-
type: 'string',
|
|
26
|
-
description: 'What is wrong, in one sentence a developer could act on.',
|
|
27
|
-
},
|
|
28
|
-
evidence: {
|
|
29
|
-
type: 'string',
|
|
30
|
-
description: 'What in the screenshot or element list shows this is a defect.',
|
|
31
|
-
},
|
|
32
|
-
severity: { type: 'string', enum: SEVERITIES },
|
|
33
|
-
ref: {
|
|
34
|
-
type: ['string', 'null'],
|
|
35
|
-
description: 'The handle of the single element the annotation should point at, e.g. "e12". ' +
|
|
36
|
-
'Null only if no listed element corresponds to the problem.',
|
|
37
|
-
},
|
|
38
|
-
region: {
|
|
39
|
-
type: ['object', 'null'],
|
|
40
|
-
description: 'Fallback used only when ref is null: the area of the viewport to box, in percentages.',
|
|
41
|
-
properties: {
|
|
42
|
-
xPct: { type: 'number' },
|
|
43
|
-
yPct: { type: 'number' },
|
|
44
|
-
widthPct: { type: 'number' },
|
|
45
|
-
heightPct: { type: 'number' },
|
|
46
|
-
},
|
|
47
|
-
required: ['xPct', 'yPct', 'widthPct', 'heightPct'],
|
|
48
|
-
additionalProperties: false,
|
|
49
|
-
},
|
|
50
|
-
},
|
|
51
|
-
required: ['description', 'evidence', 'severity', 'ref', 'region'],
|
|
52
|
-
additionalProperties: false,
|
|
53
|
-
},
|
|
54
|
-
},
|
|
55
|
-
},
|
|
56
|
-
required: ['findings'],
|
|
57
|
-
additionalProperties: false,
|
|
58
|
-
};
|
|
59
|
-
const SYSTEM_PROMPT = `You are doing exploratory visual testing of a web page.
|
|
60
|
-
|
|
61
|
-
You get a screenshot and a list of the page's elements. Every element has a handle
|
|
62
|
-
like [e12], its tag, its accessible name, and its bounding box in CSS pixels
|
|
63
|
-
(x,y width x height) relative to the viewport.
|
|
64
|
-
|
|
65
|
-
Report defects a developer would accept as real bugs: overlapping or clipped
|
|
66
|
-
content, misalignment, text overflowing its container, invisible or unreadable
|
|
67
|
-
text, controls positioned off-screen or outside their container, broken images,
|
|
68
|
-
inconsistent spacing that breaks an otherwise regular rhythm, and anything that
|
|
69
|
-
contradicts the ticket.
|
|
70
|
-
|
|
71
|
-
Do not report subjective styling preferences, and do not invent defects. If the
|
|
72
|
-
page looks correct, return an empty findings array.
|
|
73
|
-
|
|
74
|
-
For each finding, set "ref" to the handle of the single element the annotation
|
|
75
|
-
should point at - the element that is wrong, not its container, and not the
|
|
76
|
-
element it collides with. The bounding boxes in the element list are the best
|
|
77
|
-
evidence for geometric bugs: overlapping boxes, boxes extending past a parent,
|
|
78
|
-
and boxes with a zero or negative gap are all visible in the numbers.
|
|
79
|
-
|
|
80
|
-
Only set "ref" to null when no listed element corresponds to the problem - for
|
|
81
|
-
example whitespace where something should be. In that case fill in "region"
|
|
82
|
-
with the area to box, as percentages of the viewport.
|
|
83
|
-
|
|
84
|
-
Always call the report_findings tool exactly once, even when you find nothing.`;
|
|
85
|
-
export class ClaudeReasoner {
|
|
86
|
-
client;
|
|
87
|
-
model;
|
|
88
|
-
maxTokens;
|
|
89
|
-
effort;
|
|
90
|
-
constructor(options = {}) {
|
|
91
|
-
this.client = options.client ?? new Anthropic();
|
|
92
|
-
this.model = options.model ?? 'claude-opus-5';
|
|
93
|
-
this.maxTokens = options.maxTokens ?? 16000;
|
|
94
|
-
this.effort = options.effort ?? 'high';
|
|
95
|
-
}
|
|
96
|
-
async propose(input) {
|
|
97
|
-
const stream = this.client.messages.stream({
|
|
98
|
-
model: this.model,
|
|
99
|
-
max_tokens: this.maxTokens,
|
|
100
|
-
system: SYSTEM_PROMPT,
|
|
101
|
-
thinking: { type: 'adaptive' },
|
|
102
|
-
output_config: { effort: this.effort },
|
|
103
|
-
tools: [
|
|
104
|
-
{
|
|
105
|
-
name: 'report_findings',
|
|
106
|
-
description: 'Report every defect found on the page.',
|
|
107
|
-
input_schema: FINDING_SCHEMA,
|
|
108
|
-
strict: true,
|
|
109
|
-
},
|
|
110
|
-
],
|
|
111
|
-
messages: [
|
|
112
|
-
{
|
|
113
|
-
role: 'user',
|
|
114
|
-
content: [
|
|
115
|
-
{
|
|
116
|
-
type: 'image',
|
|
117
|
-
source: {
|
|
118
|
-
type: 'base64',
|
|
119
|
-
media_type: 'image/png',
|
|
120
|
-
data: input.screenshotPng.toString('base64'),
|
|
121
|
-
},
|
|
122
|
-
},
|
|
123
|
-
{
|
|
124
|
-
type: 'text',
|
|
125
|
-
text: `URL: ${input.url}\n\n` +
|
|
126
|
-
`${describeIntent(input.ticket)}\n\n` +
|
|
127
|
-
`Elements on the page:\n${renderInventory(input.inventory)}`,
|
|
128
|
-
},
|
|
129
|
-
],
|
|
130
|
-
},
|
|
131
|
-
],
|
|
132
|
-
});
|
|
133
|
-
const message = await stream.finalMessage();
|
|
134
|
-
if (message.stop_reason === 'refusal') {
|
|
135
|
-
throw new Error(`Model declined the request (${message.stop_details?.category ?? 'unknown'}).`);
|
|
136
|
-
}
|
|
137
|
-
const call = message.content.find((block) => block.type === 'tool_use' && block.name === 'report_findings');
|
|
138
|
-
if (!call) {
|
|
139
|
-
const text = message.content
|
|
140
|
-
.filter((b) => b.type === 'text')
|
|
141
|
-
.map((b) => b.text)
|
|
142
|
-
.join(' ');
|
|
143
|
-
throw new Error(`Model did not call report_findings (stop_reason: ${message.stop_reason}). Said: ${text.slice(0, 300)}`);
|
|
144
|
-
}
|
|
145
|
-
// Tool inputs are already parsed objects, but escaping varies by model, so
|
|
146
|
-
// never string-match on them.
|
|
147
|
-
const parsed = call.input;
|
|
148
|
-
return normaliseFindings(parsed.findings);
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
/**
|
|
152
|
-
* With no ticket there is nothing to contradict, so the model is told to judge
|
|
153
|
-
* the page on its own terms rather than being handed an empty requirement it
|
|
154
|
-
* might try to reason against.
|
|
155
|
-
*/
|
|
156
|
-
function describeIntent(ticket) {
|
|
157
|
-
const trimmed = ticket.trim();
|
|
158
|
-
return trimmed
|
|
159
|
-
? `Ticket / what this page is supposed to do:\n${trimmed}`
|
|
160
|
-
: 'No ticket was supplied. Judge the page on its own terms: report anything that ' +
|
|
161
|
-
'looks broken, misaligned, unreadable, clipped, or otherwise unintentional.';
|
|
162
|
-
}
|
|
163
|
-
function normaliseFindings(raw) {
|
|
164
|
-
if (!Array.isArray(raw))
|
|
165
|
-
return [];
|
|
166
|
-
const findings = [];
|
|
167
|
-
for (const entry of raw) {
|
|
168
|
-
if (typeof entry !== 'object' || entry === null)
|
|
169
|
-
continue;
|
|
170
|
-
const record = entry;
|
|
171
|
-
const severity = SEVERITIES.includes(record.severity)
|
|
172
|
-
? record.severity
|
|
173
|
-
: 'minor';
|
|
174
|
-
findings.push({
|
|
175
|
-
description: String(record.description ?? '').trim(),
|
|
176
|
-
evidence: String(record.evidence ?? '').trim(),
|
|
177
|
-
severity,
|
|
178
|
-
ref: typeof record.ref === 'string' && record.ref ? record.ref : null,
|
|
179
|
-
region: isRegion(record.region) ? record.region : null,
|
|
180
|
-
});
|
|
181
|
-
}
|
|
182
|
-
return findings.filter((f) => f.description.length > 0);
|
|
183
|
-
}
|
|
184
|
-
function isRegion(value) {
|
|
185
|
-
if (typeof value !== 'object' || value === null)
|
|
186
|
-
return false;
|
|
187
|
-
const r = value;
|
|
188
|
-
return (typeof r.xPct === 'number' &&
|
|
189
|
-
typeof r.yPct === 'number' &&
|
|
190
|
-
typeof r.widthPct === 'number' &&
|
|
191
|
-
typeof r.heightPct === 'number');
|
|
192
|
-
}
|
package/dist/cli.d.ts
DELETED
package/dist/cli.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}
|
package/dist/cli.js
DELETED
|
@@ -1,84 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import { writeFile } from 'node:fs/promises';
|
|
3
|
-
import { parseArgs } from 'node:util';
|
|
4
|
-
import { captureAnnotated } from './index.js';
|
|
5
|
-
const USAGE = `
|
|
6
|
-
Usage: cypress-annotate --url <url> --selector <css> [options]
|
|
7
|
-
|
|
8
|
-
--url <url> Page to open (http(s):// or file://).
|
|
9
|
-
--selector <css> Element to annotate. Repeatable.
|
|
10
|
-
Use ">>>" to pierce iframes: "iframe#pay >>> .btn".
|
|
11
|
-
--label <text> Label for the matching --selector. Repeatable.
|
|
12
|
-
--out <path> Output PNG. Default: annotated.png
|
|
13
|
-
--mode <m> viewport | fullPage | element. Default: viewport
|
|
14
|
-
--shape <s> box | circle | arrow. Default: box
|
|
15
|
-
--color <css> Outline colour. Default: #FF3B30
|
|
16
|
-
--dim <0..1> Darken everything outside the annotation. Default: 0
|
|
17
|
-
--width <px> Viewport width, CSS px. Default: 1280
|
|
18
|
-
--height <px> Viewport height, CSS px. Default: 800
|
|
19
|
-
--dpr <n> Device scale factor. Default: 2
|
|
20
|
-
--crop-padding <px> Context to keep around the element in 'element' mode.
|
|
21
|
-
--json Print the measurement report as JSON.
|
|
22
|
-
`.trim();
|
|
23
|
-
const { values } = parseArgs({
|
|
24
|
-
options: {
|
|
25
|
-
url: { type: 'string' },
|
|
26
|
-
selector: { type: 'string', multiple: true, default: [] },
|
|
27
|
-
label: { type: 'string', multiple: true, default: [] },
|
|
28
|
-
out: { type: 'string', default: 'annotated.png' },
|
|
29
|
-
mode: { type: 'string', default: 'viewport' },
|
|
30
|
-
shape: { type: 'string', default: 'box' },
|
|
31
|
-
color: { type: 'string' },
|
|
32
|
-
dim: { type: 'string' },
|
|
33
|
-
width: { type: 'string', default: '1280' },
|
|
34
|
-
height: { type: 'string', default: '800' },
|
|
35
|
-
dpr: { type: 'string', default: '2' },
|
|
36
|
-
'crop-padding': { type: 'string' },
|
|
37
|
-
json: { type: 'boolean', default: false },
|
|
38
|
-
help: { type: 'boolean', default: false },
|
|
39
|
-
},
|
|
40
|
-
allowPositionals: false,
|
|
41
|
-
});
|
|
42
|
-
if (values.help || !values.url || values.selector.length === 0) {
|
|
43
|
-
console.log(USAGE);
|
|
44
|
-
process.exit(values.help ? 0 : 1);
|
|
45
|
-
}
|
|
46
|
-
const annotations = values.selector.map((selector, i) => ({
|
|
47
|
-
selector: selector.includes('>>>') ? selector.split('>>>').map((s) => s.trim()) : selector,
|
|
48
|
-
label: values.label[i],
|
|
49
|
-
shape: values.shape,
|
|
50
|
-
}));
|
|
51
|
-
/**
|
|
52
|
-
* A CLI should print what went wrong and nothing else. Without this, a missing
|
|
53
|
-
* playwright surfaces as an unhandled rejection: the useful sentence buried in
|
|
54
|
-
* a Node module-loader stack, plus the whole `cause` chain printed underneath.
|
|
55
|
-
*/
|
|
56
|
-
function fail(error) {
|
|
57
|
-
console.error(error instanceof Error ? error.message : String(error));
|
|
58
|
-
process.exit(1);
|
|
59
|
-
}
|
|
60
|
-
const result = await captureAnnotated(values.url, annotations, {
|
|
61
|
-
mode: values.mode,
|
|
62
|
-
viewport: { width: Number(values.width), height: Number(values.height) },
|
|
63
|
-
deviceScaleFactor: Number(values.dpr),
|
|
64
|
-
cropPadding: values['crop-padding'] ? Number(values['crop-padding']) : undefined,
|
|
65
|
-
style: {
|
|
66
|
-
...(values.color ? { color: values.color } : {}),
|
|
67
|
-
...(values.dim ? { dimOutside: Number(values.dim) } : {}),
|
|
68
|
-
},
|
|
69
|
-
}).catch(fail);
|
|
70
|
-
await writeFile(values.out, result.image).catch(fail);
|
|
71
|
-
if (values.json) {
|
|
72
|
-
console.log(JSON.stringify({ out: values.out, width: result.width, height: result.height, metrics: result.metrics, elements: result.elements, drawnRects: result.drawnRects, warnings: result.warnings }, null, 2));
|
|
73
|
-
}
|
|
74
|
-
else {
|
|
75
|
-
console.log(`Wrote ${values.out} (${result.width}x${result.height})`);
|
|
76
|
-
for (const element of result.elements) {
|
|
77
|
-
const r = result.drawnRects[result.elements.indexOf(element)];
|
|
78
|
-
console.log(` ${element.selector} -> <${element.tagName}> ` +
|
|
79
|
-
`css ${element.viewportRect.width.toFixed(0)}x${element.viewportRect.height.toFixed(0)} ` +
|
|
80
|
-
`@ image ${r ? `${r.x.toFixed(0)},${r.y.toFixed(0)}` : '?'}`);
|
|
81
|
-
}
|
|
82
|
-
for (const warning of result.warnings)
|
|
83
|
-
console.log(` ! ${warning}`);
|
|
84
|
-
}
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
-
import type { FailureRecord } from './failure-report.js';
|
|
3
|
-
/**
|
|
4
|
-
* Claude fallback for the failures registerFailureCapture could not resolve
|
|
5
|
-
* deterministically - no selector recovered, or the recovered selector matched
|
|
6
|
-
* nothing live. Deliberately a separate, narrower task from finder.ts's
|
|
7
|
-
* open-ended bug hunting: the failure is already known (err.message says so);
|
|
8
|
-
* the only open question is *where on screen it shows up*, from a fixed list
|
|
9
|
-
* of candidate elements captured live at failure time. That's a smaller,
|
|
10
|
-
* cheaper, more constrained ask than "find bugs on this page".
|
|
11
|
-
*/
|
|
12
|
-
export interface FailureExplanation {
|
|
13
|
-
/** One of the inventory selectors from the record, or null if none fit. */
|
|
14
|
-
selector: string | null;
|
|
15
|
-
label: string;
|
|
16
|
-
/** Fallback region (percent of viewport) when no inventory item fits. */
|
|
17
|
-
region: {
|
|
18
|
-
xPct: number;
|
|
19
|
-
yPct: number;
|
|
20
|
-
widthPct: number;
|
|
21
|
-
heightPct: number;
|
|
22
|
-
} | null;
|
|
23
|
-
}
|
|
24
|
-
export interface FailureExplainer {
|
|
25
|
-
explain(record: FailureRecord): Promise<FailureExplanation>;
|
|
26
|
-
}
|
|
27
|
-
export interface ClaudeFailureExplainerOptions {
|
|
28
|
-
model?: string;
|
|
29
|
-
effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
30
|
-
client?: Anthropic;
|
|
31
|
-
}
|
|
32
|
-
export declare class ClaudeFailureExplainer implements FailureExplainer {
|
|
33
|
-
private readonly client;
|
|
34
|
-
private readonly model;
|
|
35
|
-
private readonly effort;
|
|
36
|
-
constructor(options?: ClaudeFailureExplainerOptions);
|
|
37
|
-
explain(record: FailureRecord): Promise<FailureExplanation>;
|
|
38
|
-
}
|
|
39
|
-
/** For tests and offline runs - no API call. */
|
|
40
|
-
export declare class ReplayFailureExplainer implements FailureExplainer {
|
|
41
|
-
private readonly explanation;
|
|
42
|
-
constructor(explanation: FailureExplanation);
|
|
43
|
-
explain(): Promise<FailureExplanation>;
|
|
44
|
-
}
|
|
45
|
-
//# sourceMappingURL=failure-explainer.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"failure-explainer.d.ts","sourceRoot":"","sources":["../../src/cypress/failure-explainer.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD;;;;;;;;GAQG;AAEH,MAAM,WAAW,kBAAkB;IACjC,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;CACpF;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAC7D;AA6DD,MAAM,WAAW,6BAA6B;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,OAAO,GAAG,KAAK,CAAC;IACrD,MAAM,CAAC,EAAE,SAAS,CAAC;CACpB;AAED,qBAAa,sBAAuB,YAAW,gBAAgB;IAC7D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA8C;gBAEzD,OAAO,GAAE,6BAAkC;IAMjD,OAAO,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAoDlE;AAED,gDAAgD;AAChD,qBAAa,sBAAuB,YAAW,gBAAgB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW;gBAAX,WAAW,EAAE,kBAAkB;IACtD,OAAO,IAAI,OAAO,CAAC,kBAAkB,CAAC;CAG7C"}
|
|
@@ -1,123 +0,0 @@
|
|
|
1
|
-
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
-
const SCHEMA = {
|
|
3
|
-
type: 'object',
|
|
4
|
-
properties: {
|
|
5
|
-
selector: {
|
|
6
|
-
type: ['string', 'null'],
|
|
7
|
-
description: 'The "selector" value of the single inventory element that best shows where this failure is ' +
|
|
8
|
-
'visible on screen. Null only if no listed element corresponds to it.',
|
|
9
|
-
},
|
|
10
|
-
label: {
|
|
11
|
-
type: 'string',
|
|
12
|
-
description: 'One short sentence for the annotation, explaining the failure in plain terms.',
|
|
13
|
-
},
|
|
14
|
-
region: {
|
|
15
|
-
type: ['object', 'null'],
|
|
16
|
-
description: 'Fallback used only when selector is null: the area to box, in percentages of the viewport.',
|
|
17
|
-
properties: {
|
|
18
|
-
xPct: { type: 'number' },
|
|
19
|
-
yPct: { type: 'number' },
|
|
20
|
-
widthPct: { type: 'number' },
|
|
21
|
-
heightPct: { type: 'number' },
|
|
22
|
-
},
|
|
23
|
-
required: ['xPct', 'yPct', 'widthPct', 'heightPct'],
|
|
24
|
-
additionalProperties: false,
|
|
25
|
-
},
|
|
26
|
-
},
|
|
27
|
-
required: ['selector', 'label', 'region'],
|
|
28
|
-
additionalProperties: false,
|
|
29
|
-
};
|
|
30
|
-
const SYSTEM_PROMPT = `A Cypress test failed. You are given the screenshot at the moment of failure,
|
|
31
|
-
the test's own error message, and a list of candidate elements on the page with their
|
|
32
|
-
tag, visible text, and bounding box in CSS pixels.
|
|
33
|
-
|
|
34
|
-
This is not exploratory bug hunting - the failure is already known and described by the
|
|
35
|
-
error message. Your only job is to say where on screen it is visible, by picking the
|
|
36
|
-
single best-matching element from the list, and to write one short, plain-language
|
|
37
|
-
sentence explaining it for someone triaging CI failures who has not read the error.
|
|
38
|
-
|
|
39
|
-
Prefer an element whose visible text or position plausibly explains the error message
|
|
40
|
-
over one that merely has a similar name. If nothing in the list corresponds to the
|
|
41
|
-
failure - for example the error describes something absent, like a modal that should
|
|
42
|
-
have appeared - set selector to null and give a region instead, as percentages of the
|
|
43
|
-
viewport, for roughly where it was expected.
|
|
44
|
-
|
|
45
|
-
Always call the report tool exactly once.`;
|
|
46
|
-
function renderInventory(inventory) {
|
|
47
|
-
if (!inventory)
|
|
48
|
-
return '(no candidate elements captured)';
|
|
49
|
-
return inventory
|
|
50
|
-
.map((item) => {
|
|
51
|
-
const r = item.rect;
|
|
52
|
-
const box = r ? `${Math.round(r.x)},${Math.round(r.y)} ${Math.round(r.width)}x${Math.round(r.height)}` : '?';
|
|
53
|
-
const text = item.text ? ` "${item.text}"` : '';
|
|
54
|
-
return `[${item.selector}] <${item.tag ?? '?'}>${text} @ ${box}`;
|
|
55
|
-
})
|
|
56
|
-
.join('\n');
|
|
57
|
-
}
|
|
58
|
-
export class ClaudeFailureExplainer {
|
|
59
|
-
client;
|
|
60
|
-
model;
|
|
61
|
-
effort;
|
|
62
|
-
constructor(options = {}) {
|
|
63
|
-
this.client = options.client ?? new Anthropic();
|
|
64
|
-
this.model = options.model ?? 'claude-opus-5';
|
|
65
|
-
this.effort = options.effort ?? 'medium';
|
|
66
|
-
}
|
|
67
|
-
async explain(record) {
|
|
68
|
-
const { readFile } = await import('node:fs/promises');
|
|
69
|
-
const screenshot = await readFile(record.screenshotPath);
|
|
70
|
-
const stream = this.client.messages.stream({
|
|
71
|
-
model: this.model,
|
|
72
|
-
max_tokens: 4096,
|
|
73
|
-
system: SYSTEM_PROMPT,
|
|
74
|
-
thinking: { type: 'adaptive' },
|
|
75
|
-
output_config: { effort: this.effort },
|
|
76
|
-
tools: [
|
|
77
|
-
{
|
|
78
|
-
name: 'report_location',
|
|
79
|
-
description: 'Report where the known failure is visible on screen.',
|
|
80
|
-
input_schema: SCHEMA,
|
|
81
|
-
strict: true,
|
|
82
|
-
},
|
|
83
|
-
],
|
|
84
|
-
messages: [
|
|
85
|
-
{
|
|
86
|
-
role: 'user',
|
|
87
|
-
content: [
|
|
88
|
-
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: screenshot.toString('base64') } },
|
|
89
|
-
{
|
|
90
|
-
type: 'text',
|
|
91
|
-
text: `Test: ${record.test}\n` +
|
|
92
|
-
`Error: ${record.errMessage}\n\n` +
|
|
93
|
-
`Candidate elements:\n${renderInventory(record.inventory)}`,
|
|
94
|
-
},
|
|
95
|
-
],
|
|
96
|
-
},
|
|
97
|
-
],
|
|
98
|
-
});
|
|
99
|
-
const message = await stream.finalMessage();
|
|
100
|
-
if (message.stop_reason === 'refusal') {
|
|
101
|
-
throw new Error(`Model declined (${message.stop_details?.category ?? 'unknown'}).`);
|
|
102
|
-
}
|
|
103
|
-
const call = message.content.find((b) => b.type === 'tool_use' && b.name === 'report_location');
|
|
104
|
-
if (!call)
|
|
105
|
-
throw new Error(`Model did not call report_location (stop_reason: ${message.stop_reason}).`);
|
|
106
|
-
const input = call.input;
|
|
107
|
-
return {
|
|
108
|
-
selector: typeof input.selector === 'string' ? input.selector : null,
|
|
109
|
-
label: typeof input.label === 'string' && input.label ? input.label : record.label,
|
|
110
|
-
region: input.region ?? null,
|
|
111
|
-
};
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
/** For tests and offline runs - no API call. */
|
|
115
|
-
export class ReplayFailureExplainer {
|
|
116
|
-
explanation;
|
|
117
|
-
constructor(explanation) {
|
|
118
|
-
this.explanation = explanation;
|
|
119
|
-
}
|
|
120
|
-
async explain() {
|
|
121
|
-
return this.explanation;
|
|
122
|
-
}
|
|
123
|
-
}
|
package/dist/finder.d.ts
DELETED
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
import type { Page } from 'playwright';
|
|
2
|
-
import { type InventoryItem } from './inventory.js';
|
|
3
|
-
import type { Annotation, Region } from './types.js';
|
|
4
|
-
export type Severity = 'blocker' | 'major' | 'minor' | 'cosmetic';
|
|
5
|
-
export interface BugFinding {
|
|
6
|
-
description: string;
|
|
7
|
-
/** Inventory handle, e.g. "e12". Null when the model could not pick one. */
|
|
8
|
-
ref: string | null;
|
|
9
|
-
/** Percentage fallback, used only when `ref` is null. */
|
|
10
|
-
region: Region | null;
|
|
11
|
-
severity: Severity;
|
|
12
|
-
/** What the model saw that made it call this a bug. */
|
|
13
|
-
evidence: string;
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* The reasoning step is an interface so the pipeline can be exercised with a
|
|
17
|
-
* recorded response, and so a different backend (computer use, an MCP server)
|
|
18
|
-
* can be dropped in without touching the annotation code.
|
|
19
|
-
*
|
|
20
|
-
* This file deliberately has no dependency on `@anthropic-ai/sdk` - see
|
|
21
|
-
* claude-reasoner.ts, which implements this interface and is the only file
|
|
22
|
-
* that does. Keeping them separate means importing anything here (from the
|
|
23
|
-
* package's root barrel) never forces the SDK to be installed.
|
|
24
|
-
*/
|
|
25
|
-
export interface Reasoner {
|
|
26
|
-
propose(input: ReasonerInput): Promise<BugFinding[]>;
|
|
27
|
-
}
|
|
28
|
-
export interface ReasonerInput {
|
|
29
|
-
screenshotPng: Buffer;
|
|
30
|
-
inventory: InventoryItem[];
|
|
31
|
-
ticket: string;
|
|
32
|
-
url: string;
|
|
33
|
-
}
|
|
34
|
-
/** A reasoner backed by a recorded response, for tests and offline runs. */
|
|
35
|
-
export declare class ReplayReasoner implements Reasoner {
|
|
36
|
-
private readonly findings;
|
|
37
|
-
constructor(findings: BugFinding[]);
|
|
38
|
-
propose(): Promise<BugFinding[]>;
|
|
39
|
-
}
|
|
40
|
-
export declare const SEVERITY_COLORS: Record<Severity, string>;
|
|
41
|
-
export interface ResolvedFinding {
|
|
42
|
-
finding: BugFinding;
|
|
43
|
-
annotation: Annotation;
|
|
44
|
-
/** Present when the finding resolved through an inventory handle. */
|
|
45
|
-
item?: InventoryItem;
|
|
46
|
-
/**
|
|
47
|
-
* Stable identity for de-duplicating the same bug seen in overlapping
|
|
48
|
-
* viewport passes. Inventory handles are re-issued on every pass, so this is
|
|
49
|
-
* keyed on the readable selector instead.
|
|
50
|
-
*/
|
|
51
|
-
key: string;
|
|
52
|
-
}
|
|
53
|
-
export interface ResolvedFindings {
|
|
54
|
-
resolved: ResolvedFinding[];
|
|
55
|
-
/** Findings that named neither a valid ref nor a usable region. */
|
|
56
|
-
unresolved: {
|
|
57
|
-
finding: BugFinding;
|
|
58
|
-
reason: string;
|
|
59
|
-
}[];
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Turn model findings into annotations. A ref is looked up in the inventory,
|
|
63
|
-
* which is what guarantees the selector resolves; a region is passed through;
|
|
64
|
-
* anything else is reported as unresolved rather than silently dropped.
|
|
65
|
-
*/
|
|
66
|
-
export declare function findingsToAnnotations(findings: BugFinding[], inventory: InventoryItem[]): ResolvedFindings;
|
|
67
|
-
export declare function gatherReasonerInput(page: Page, ticket: string, viewportOnly?: boolean): Promise<ReasonerInput>;
|
|
68
|
-
//# sourceMappingURL=finder.d.ts.map
|
package/dist/finder.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"finder.d.ts","sourceRoot":"","sources":["../src/finder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAoB,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAErD,MAAM,MAAM,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAC;AAElE,MAAM,WAAW,UAAU;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,yDAAyD;IACzD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,QAAQ,CAAC;IACnB,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,QAAQ;IACvB,OAAO,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;CACb;AAED,4EAA4E;AAC5E,qBAAa,cAAe,YAAW,QAAQ;IACjC,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,UAAU,EAAE;IAC7C,OAAO,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;CAGvC;AAED,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAKpD,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,UAAU,CAAC;IACvB,qEAAqE;IACrE,IAAI,CAAC,EAAE,aAAa,CAAC;IACrB;;;;OAIG;IACH,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,mEAAmE;IACnE,UAAU,EAAE;QAAE,OAAO,EAAE,UAAU,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACvD;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,UAAU,EAAE,EACtB,SAAS,EAAE,aAAa,EAAE,GACzB,gBAAgB,CAyClB;AAED,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,MAAM,EACd,YAAY,UAAO,GAClB,OAAO,CAAC,aAAa,CAAC,CAIxB"}
|
package/dist/finder.js
DELETED
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
import { collectInventory } from './inventory.js';
|
|
2
|
-
/** A reasoner backed by a recorded response, for tests and offline runs. */
|
|
3
|
-
export class ReplayReasoner {
|
|
4
|
-
findings;
|
|
5
|
-
constructor(findings) {
|
|
6
|
-
this.findings = findings;
|
|
7
|
-
}
|
|
8
|
-
async propose() {
|
|
9
|
-
return this.findings;
|
|
10
|
-
}
|
|
11
|
-
}
|
|
12
|
-
export const SEVERITY_COLORS = {
|
|
13
|
-
blocker: '#FF2D2D',
|
|
14
|
-
major: '#FF6A00',
|
|
15
|
-
minor: '#F5A623',
|
|
16
|
-
cosmetic: '#3D7BFF',
|
|
17
|
-
};
|
|
18
|
-
/**
|
|
19
|
-
* Turn model findings into annotations. A ref is looked up in the inventory,
|
|
20
|
-
* which is what guarantees the selector resolves; a region is passed through;
|
|
21
|
-
* anything else is reported as unresolved rather than silently dropped.
|
|
22
|
-
*/
|
|
23
|
-
export function findingsToAnnotations(findings, inventory) {
|
|
24
|
-
const byRef = new Map(inventory.map((item) => [item.ref, item]));
|
|
25
|
-
const resolved = [];
|
|
26
|
-
const unresolved = [];
|
|
27
|
-
for (const finding of findings) {
|
|
28
|
-
const style = { color: SEVERITY_COLORS[finding.severity] };
|
|
29
|
-
const label = `${finding.severity}: ${finding.description}`;
|
|
30
|
-
if (finding.ref) {
|
|
31
|
-
const item = byRef.get(finding.ref);
|
|
32
|
-
if (item) {
|
|
33
|
-
resolved.push({
|
|
34
|
-
finding,
|
|
35
|
-
annotation: { selector: item.target, label, style },
|
|
36
|
-
item,
|
|
37
|
-
key: `sel:${item.reportSelector}`,
|
|
38
|
-
});
|
|
39
|
-
continue;
|
|
40
|
-
}
|
|
41
|
-
// A hallucinated handle is recoverable if a region came with it.
|
|
42
|
-
if (!finding.region) {
|
|
43
|
-
unresolved.push({ finding, reason: `ref "${finding.ref}" is not in the inventory` });
|
|
44
|
-
continue;
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
if (finding.region) {
|
|
48
|
-
const r = finding.region;
|
|
49
|
-
resolved.push({
|
|
50
|
-
finding,
|
|
51
|
-
annotation: { region: r, label, style },
|
|
52
|
-
key: `region:${Math.round(r.xPct)},${Math.round(r.yPct)},${Math.round(r.widthPct)},${Math.round(r.heightPct)}`,
|
|
53
|
-
});
|
|
54
|
-
continue;
|
|
55
|
-
}
|
|
56
|
-
unresolved.push({ finding, reason: 'no ref and no region' });
|
|
57
|
-
}
|
|
58
|
-
return { resolved, unresolved };
|
|
59
|
-
}
|
|
60
|
-
export async function gatherReasonerInput(page, ticket, viewportOnly = true) {
|
|
61
|
-
const inventory = await collectInventory(page, { viewportOnly });
|
|
62
|
-
const screenshotPng = await page.screenshot({ animations: 'disabled', caret: 'hide', type: 'png' });
|
|
63
|
-
return { screenshotPng, inventory, ticket, url: page.url() };
|
|
64
|
-
}
|
package/dist/inventory.d.ts
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
import type { Page } from 'playwright';
|
|
2
|
-
import type { CssRect } from './types.js';
|
|
3
|
-
export interface InventoryItem {
|
|
4
|
-
/** Handle the model refers to, e.g. "e12". */
|
|
5
|
-
ref: string;
|
|
6
|
-
/**
|
|
7
|
-
* Selector chain that is guaranteed to resolve, because the attribute it
|
|
8
|
-
* matches was injected during collection.
|
|
9
|
-
*/
|
|
10
|
-
target: string[];
|
|
11
|
-
/** Readable selector for the bug report, derived from ids/test-ids/structure. */
|
|
12
|
-
reportSelector: string;
|
|
13
|
-
tag: string;
|
|
14
|
-
role: string | null;
|
|
15
|
-
/** Accessible-ish name: aria-label, alt, placeholder, value, or trimmed text. */
|
|
16
|
-
name: string;
|
|
17
|
-
rect: CssRect;
|
|
18
|
-
inViewport: boolean;
|
|
19
|
-
/** Empty for the main document. */
|
|
20
|
-
framePath: string[];
|
|
21
|
-
}
|
|
22
|
-
export interface InventoryOptions {
|
|
23
|
-
/** Cap on items sent to the model, to bound token cost. Default 120. */
|
|
24
|
-
limit?: number;
|
|
25
|
-
/** Ignore anything smaller than this many CSS px on either axis. Default 8. */
|
|
26
|
-
minSize?: number;
|
|
27
|
-
/** Only collect elements currently intersecting the viewport. Default false. */
|
|
28
|
-
viewportOnly?: boolean;
|
|
29
|
-
}
|
|
30
|
-
export declare function collectInventory(page: Page, options?: InventoryOptions): Promise<InventoryItem[]>;
|
|
31
|
-
/** Compact text rendering of the inventory for the model prompt. */
|
|
32
|
-
export declare function renderInventory(items: InventoryItem[]): string;
|
|
33
|
-
//# sourceMappingURL=inventory.d.ts.map
|
package/dist/inventory.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"inventory.d.ts","sourceRoot":"","sources":["../src/inventory.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAS,IAAI,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,WAAW,aAAa;IAC5B,8CAA8C;IAC9C,GAAG,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,iFAAiF;IACjF,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,UAAU,EAAE,OAAO,CAAC;IACpB,mCAAmC;IACnC,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,wEAAwE;IACxE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gFAAgF;IAChF,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AA8DD,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,IAAI,EACV,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,aAAa,EAAE,CAAC,CAkC1B;AAED,oEAAoE;AACpE,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,MAAM,CAW9D"}
|