@kintsugi-ai/core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +16 -0
- package/dist/agentConfig.d.ts +17 -0
- package/dist/agentConfig.js +91 -0
- package/dist/agentInstructions.d.ts +11 -0
- package/dist/agentInstructions.js +58 -0
- package/dist/auth.d.ts +35 -0
- package/dist/auth.js +121 -1
- package/dist/baseline.d.ts +9 -0
- package/dist/baseline.js +47 -0
- package/dist/capturer.js +6 -0
- package/dist/classifier-state.d.ts +13 -0
- package/dist/classifier-state.js +39 -0
- package/dist/comparator.d.ts +7 -0
- package/dist/comparator.js +6 -0
- package/dist/config.js +0 -6
- package/dist/escalation.d.ts +55 -0
- package/dist/escalation.js +99 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +5 -0
- package/dist/net.d.ts +7 -0
- package/dist/net.js +20 -0
- package/dist/setupPrompt.js +4 -2
- package/dist/types.d.ts +9 -4
- package/package.json +16 -1
- package/dist/auth.d.ts.map +0 -1
- package/dist/auth.js.map +0 -1
- package/dist/baseline.d.ts.map +0 -1
- package/dist/baseline.js.map +0 -1
- package/dist/benchmark.d.ts.map +0 -1
- package/dist/benchmark.js.map +0 -1
- package/dist/capturer.d.ts.map +0 -1
- package/dist/capturer.js.map +0 -1
- package/dist/classifier.d.ts.map +0 -1
- package/dist/classifier.js.map +0 -1
- package/dist/comparator.d.ts.map +0 -1
- package/dist/comparator.js.map +0 -1
- package/dist/comparator.test.d.ts +0 -2
- package/dist/comparator.test.d.ts.map +0 -1
- package/dist/comparator.test.js +0 -73
- package/dist/comparator.test.js.map +0 -1
- package/dist/config.d.ts.map +0 -1
- package/dist/config.js.map +0 -1
- package/dist/config.test.d.ts +0 -2
- package/dist/config.test.d.ts.map +0 -1
- package/dist/config.test.js +0 -67
- package/dist/config.test.js.map +0 -1
- package/dist/domdiff.d.ts.map +0 -1
- package/dist/domdiff.js.map +0 -1
- package/dist/feedback.d.ts.map +0 -1
- package/dist/feedback.js.map +0 -1
- package/dist/flowid.d.ts.map +0 -1
- package/dist/flowid.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/recorder.d.ts.map +0 -1
- package/dist/recorder.js.map +0 -1
- package/dist/setupPrompt.d.ts.map +0 -1
- package/dist/setupPrompt.js.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/vision.d.ts +0 -34
- package/dist/vision.d.ts.map +0 -1
- package/dist/vision.js +0 -139
- package/dist/vision.js.map +0 -1
- package/src/auth.ts +0 -195
- package/src/baseline.ts +0 -125
- package/src/benchmark.ts +0 -307
- package/src/capturer.ts +0 -105
- package/src/classifier.ts +0 -258
- package/src/comparator.test.ts +0 -97
- package/src/comparator.ts +0 -217
- package/src/config.test.ts +0 -80
- package/src/config.ts +0 -83
- package/src/domdiff.ts +0 -62
- package/src/feedback.ts +0 -64
- package/src/flowid.ts +0 -14
- package/src/index.ts +0 -13
- package/src/recorder.ts +0 -96
- package/src/setupPrompt.ts +0 -26
- package/src/types.ts +0 -144
- package/tsconfig.json +0 -9
package/src/comparator.test.ts
DELETED
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
import { test, describe } from 'node:test';
|
|
2
|
-
import assert from 'node:assert/strict';
|
|
3
|
-
import {
|
|
4
|
-
createSyntheticUiPng,
|
|
5
|
-
computePixelDiff,
|
|
6
|
-
compareScreenshots,
|
|
7
|
-
diffAriaSnapshots,
|
|
8
|
-
parseVerdict,
|
|
9
|
-
DiffResult
|
|
10
|
-
} from './index.js';
|
|
11
|
-
|
|
12
|
-
describe('Kintsugi Visual Comparator Engine', () => {
|
|
13
|
-
test('identical images return 0% diff and IDENTICAL status', async () => {
|
|
14
|
-
const img1 = createSyntheticUiPng();
|
|
15
|
-
const img2 = createSyntheticUiPng();
|
|
16
|
-
|
|
17
|
-
const pixelResult = await computePixelDiff(img1, img2);
|
|
18
|
-
assert.equal(pixelResult.diffPercent, 0);
|
|
19
|
-
|
|
20
|
-
const comp = await compareScreenshots(img1, img2);
|
|
21
|
-
assert.equal(comp.result, DiffResult.IDENTICAL);
|
|
22
|
-
assert.equal(comp.pixelDiffPercent, 0);
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
test('large pixel diff without a classifier escalates as CHANGED', async () => {
|
|
26
|
-
const baseline = createSyntheticUiPng({ removeButton: false });
|
|
27
|
-
const regressed = createSyntheticUiPng({ removeButton: true });
|
|
28
|
-
|
|
29
|
-
const pixelResult = await computePixelDiff(baseline, regressed);
|
|
30
|
-
assert.ok(pixelResult.diffPercent > 0.5, 'Pixel diff should exceed 0.5%');
|
|
31
|
-
|
|
32
|
-
const comp = await compareScreenshots(baseline, regressed);
|
|
33
|
-
assert.equal(comp.result, DiffResult.CHANGED, 'Missing button should escalate as CHANGED');
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
test('sub-threshold noise is classified as MINOR', async () => {
|
|
37
|
-
const baseline = createSyntheticUiPng();
|
|
38
|
-
const withNoise = createSyntheticUiPng({ addNoise: true, noiseAmount: 0.0002 });
|
|
39
|
-
|
|
40
|
-
const comp = await compareScreenshots(baseline, withNoise);
|
|
41
|
-
assert.ok(
|
|
42
|
-
comp.result === DiffResult.IDENTICAL || comp.result === DiffResult.MINOR,
|
|
43
|
-
'Subpixel noise should be classified as IDENTICAL or MINOR, not BROKEN'
|
|
44
|
-
);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
test('aria snapshot diff reports removed and added lines', () => {
|
|
48
|
-
const baseline = [
|
|
49
|
-
'- button "Complete Purchase ($89.00)"',
|
|
50
|
-
'- text "Free shipping"',
|
|
51
|
-
].join('\n');
|
|
52
|
-
const current = [
|
|
53
|
-
'- text "Free shipping"',
|
|
54
|
-
'- textbox "Email"',
|
|
55
|
-
].join('\n');
|
|
56
|
-
|
|
57
|
-
const diff = diffAriaSnapshots(baseline, current);
|
|
58
|
-
assert.ok(diff.includes('REMOVED: - button "Complete Purchase ($89.00)"'), 'Removed button should appear as removed');
|
|
59
|
-
assert.ok(diff.includes('textbox "Email"') && diff.includes('ADDED:'), 'Added textbox should appear as added');
|
|
60
|
-
assert.ok(!diff.includes('no line-level differences'));
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
test('aria snapshot diff of identical snapshots reports no differences', () => {
|
|
64
|
-
const snap = '- button "Buy"\n- text "Hello"';
|
|
65
|
-
assert.ok(diffAriaSnapshots(snap, snap).includes('no line-level differences'));
|
|
66
|
-
});
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
describe('Classifier verdict parsing', () => {
|
|
70
|
-
test('parses the JSON verdict format', () => {
|
|
71
|
-
const v = parseVerdict('{"intentional": true, "confidence": 0.92, "reasoning": "Matches the requested recolor."}');
|
|
72
|
-
assert.equal(v.intentional, true);
|
|
73
|
-
assert.equal(v.confidence, 0.92);
|
|
74
|
-
assert.ok(v.reasoning.includes('recolor'));
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
test('parses JSON embedded in chatter', () => {
|
|
78
|
-
const v = parseVerdict('Sure! {"intentional": false, "confidence": 0.85, "reasoning": "The checkout button was removed; re-add it."} hope that helps');
|
|
79
|
-
assert.equal(v.intentional, false);
|
|
80
|
-
assert.equal(v.confidence, 0.85);
|
|
81
|
-
assert.ok(v.reasoning.includes('re-add'));
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
test('defaults missing confidence and reasoning conservatively', () => {
|
|
85
|
-
const v = parseVerdict('{"intentional": true}');
|
|
86
|
-
assert.equal(v.intentional, true);
|
|
87
|
-
assert.equal(v.confidence, 0.5);
|
|
88
|
-
assert.ok(v.reasoning.length > 0);
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
test('fails safe to not-intentional on unparseable output', () => {
|
|
92
|
-
const v = parseVerdict('I cannot tell what changed here, sorry.');
|
|
93
|
-
assert.equal(v.intentional, false);
|
|
94
|
-
assert.ok(v.confidence <= 0.7);
|
|
95
|
-
assert.ok(v.reasoning.length > 0);
|
|
96
|
-
});
|
|
97
|
-
});
|
package/src/comparator.ts
DELETED
|
@@ -1,217 +0,0 @@
|
|
|
1
|
-
import pixelmatch from 'pixelmatch';
|
|
2
|
-
import { PNG } from 'pngjs';
|
|
3
|
-
import { promises as fs } from 'fs';
|
|
4
|
-
import path from 'path';
|
|
5
|
-
import { DiffResult, type StepComparisonResult, type StepClassification, type FlowRecording, type CaptureResult, type ComparisonResult, type ClassifierConfig } from './types.js';
|
|
6
|
-
import { classifyChange, FlowLimitError } from './classifier.js';
|
|
7
|
-
import { flowFingerprint } from './flowid.js';
|
|
8
|
-
import { diffAriaSnapshots } from './domdiff.js';
|
|
9
|
-
|
|
10
|
-
/** Everything the comparator needs to consult the classifier for large diffs */
|
|
11
|
-
export interface ClassifierContext {
|
|
12
|
-
config: ClassifierConfig;
|
|
13
|
-
apiToken: string;
|
|
14
|
-
/** What the agent was asked to do in the latest turn (sharpens intentional-vs-regression) */
|
|
15
|
-
context?: string;
|
|
16
|
-
logger?: { info: (msg: string, data?: unknown) => void; warn: (msg: string, data?: unknown) => void };
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export interface CompareOptions {
|
|
20
|
-
outputDir?: string;
|
|
21
|
-
classifier?: ClassifierContext;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
export async function computePixelDiff(img1: Buffer, img2: Buffer): Promise<{ diffPercent: number; diffImage: Buffer }> {
|
|
25
|
-
const png1 = PNG.sync.read(img1);
|
|
26
|
-
const png2 = PNG.sync.read(img2);
|
|
27
|
-
|
|
28
|
-
if (png1.width !== png2.width || png1.height !== png2.height) {
|
|
29
|
-
return { diffPercent: 100, diffImage: Buffer.alloc(0) };
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
const { width, height } = png1;
|
|
33
|
-
const diff = new PNG({ width, height });
|
|
34
|
-
|
|
35
|
-
const numDiffPixels = pixelmatch(png1.data, png2.data, diff.data, width, height, { threshold: 0.1 });
|
|
36
|
-
const diffPercent = (numDiffPixels / (width * height)) * 100;
|
|
37
|
-
const diffImage = PNG.sync.write(diff);
|
|
38
|
-
|
|
39
|
-
return { diffPercent, diffImage };
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* Pixel-only verdict for one screenshot pair. Diff percentages above the
|
|
44
|
-
* classifier threshold are returned as a tentative CHANGED — compareFlow
|
|
45
|
-
* refines those with the classifier when one is configured.
|
|
46
|
-
*/
|
|
47
|
-
export async function compareScreenshots(baseline: Buffer, current: Buffer, stepIndex: number = 0, options?: CompareOptions): Promise<StepComparisonResult> {
|
|
48
|
-
const outputDir = options?.outputDir;
|
|
49
|
-
const { diffPercent, diffImage } = await computePixelDiff(baseline, current);
|
|
50
|
-
|
|
51
|
-
let result: DiffResult;
|
|
52
|
-
if (diffPercent <= 0.05) {
|
|
53
|
-
result = DiffResult.IDENTICAL;
|
|
54
|
-
} else if (diffPercent > (options?.classifier?.config.pixelDiffThreshold ?? 0.5)) {
|
|
55
|
-
result = DiffResult.CHANGED;
|
|
56
|
-
} else {
|
|
57
|
-
// Sub-threshold change: benign (anti-aliasing, subpixel shifts, dynamic content)
|
|
58
|
-
result = DiffResult.MINOR;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
let diffImagePath;
|
|
62
|
-
if (outputDir && result !== DiffResult.IDENTICAL) {
|
|
63
|
-
diffImagePath = path.join(outputDir, `diff_${stepIndex}.png`);
|
|
64
|
-
await fs.writeFile(diffImagePath, diffImage);
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
return {
|
|
68
|
-
step: stepIndex,
|
|
69
|
-
pixelDiffPercent: diffPercent,
|
|
70
|
-
ssimScore: 1, // deprecated field, kept for output compatibility
|
|
71
|
-
result,
|
|
72
|
-
diffImagePath,
|
|
73
|
-
};
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
export async function compareFlow(baseline: FlowRecording, current: CaptureResult, options?: CompareOptions): Promise<ComparisonResult> {
|
|
77
|
-
const stepResults: StepComparisonResult[] = [];
|
|
78
|
-
const len = Math.min(baseline.screenshotPaths.length, current.screenshots.length);
|
|
79
|
-
|
|
80
|
-
// Pass 1: pixel diff every step (cheap, deterministic)
|
|
81
|
-
for (let i = 0; i < len; i++) {
|
|
82
|
-
const baselineBuffer = await fs.readFile(baseline.screenshotPaths[i]);
|
|
83
|
-
const currentBuffer = current.screenshots[i];
|
|
84
|
-
stepResults.push(await compareScreenshots(baselineBuffer, currentBuffer, i, options));
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
// Pass 2: classify every oversized step — all classifier calls in parallel
|
|
88
|
-
const threshold = options?.classifier?.config.pixelDiffThreshold ?? 0.5;
|
|
89
|
-
const candidates = stepResults.filter(s => s.pixelDiffPercent > threshold);
|
|
90
|
-
const classifications = options?.classifier
|
|
91
|
-
? await classifySteps(candidates, baseline, current, options.classifier)
|
|
92
|
-
: [];
|
|
93
|
-
for (const c of classifications) {
|
|
94
|
-
const step = stepResults.find(s => s.step === c.step);
|
|
95
|
-
if (step) {
|
|
96
|
-
step.result = c.intentional && c.confidence > (options!.classifier!.config.confidenceThreshold)
|
|
97
|
-
? DiffResult.INTENTIONAL
|
|
98
|
-
: DiffResult.CHANGED;
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
// Aggregate the worst verdict across steps
|
|
103
|
-
let worstResult = DiffResult.IDENTICAL;
|
|
104
|
-
let failedStep: number | undefined;
|
|
105
|
-
for (const comp of stepResults) {
|
|
106
|
-
if (comp.result === DiffResult.CHANGED || comp.result === DiffResult.BROKEN || comp.result === DiffResult.ERROR) {
|
|
107
|
-
if (worstResult !== DiffResult.CHANGED && worstResult !== DiffResult.BROKEN && worstResult !== DiffResult.ERROR) {
|
|
108
|
-
worstResult = comp.result;
|
|
109
|
-
failedStep = comp.step;
|
|
110
|
-
}
|
|
111
|
-
} else if (comp.result === DiffResult.MINOR && worstResult === DiffResult.IDENTICAL) {
|
|
112
|
-
worstResult = DiffResult.MINOR;
|
|
113
|
-
} else if (comp.result === DiffResult.INTENTIONAL && worstResult === DiffResult.IDENTICAL) {
|
|
114
|
-
worstResult = DiffResult.INTENTIONAL;
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
if (current.errors.length > 0) {
|
|
119
|
-
worstResult = DiffResult.BROKEN;
|
|
120
|
-
failedStep = current.completedSteps;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
const worstStep = stepResults.find(s => s.step === failedStep) || stepResults[0];
|
|
124
|
-
|
|
125
|
-
// Surface the classifier's explanation in the escalation panel (semanticAnalysis
|
|
126
|
-
// is what the VS Code split-view displays).
|
|
127
|
-
const worstRegression = classifications.find(c => !c.intentional);
|
|
128
|
-
|
|
129
|
-
return {
|
|
130
|
-
result: worstResult,
|
|
131
|
-
pixelDiffPercent: worstStep?.pixelDiffPercent || 0,
|
|
132
|
-
ssimScore: worstStep?.ssimScore || 1,
|
|
133
|
-
diffImagePath: worstStep?.diffImagePath,
|
|
134
|
-
semanticAnalysis: worstRegression?.reasoning,
|
|
135
|
-
failedStep,
|
|
136
|
-
stepResults,
|
|
137
|
-
classifications: classifications.length > 0 ? classifications : undefined,
|
|
138
|
-
};
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
/**
|
|
142
|
-
* Classifies all oversized steps in parallel. Fail-safe: a missing aria
|
|
143
|
-
* snapshot or any classifier error yields a not-intentional classification
|
|
144
|
-
* so the step still escalates.
|
|
145
|
-
*/
|
|
146
|
-
async function classifySteps(
|
|
147
|
-
candidates: StepComparisonResult[],
|
|
148
|
-
baseline: FlowRecording,
|
|
149
|
-
current: CaptureResult,
|
|
150
|
-
classifier: ClassifierContext
|
|
151
|
-
): Promise<StepClassification[]> {
|
|
152
|
-
return Promise.all(candidates.map(async (candidate): Promise<StepClassification> => {
|
|
153
|
-
const i = candidate.step;
|
|
154
|
-
const currentAria = current.ariaSnapshots?.[i];
|
|
155
|
-
let baselineAria: string | undefined;
|
|
156
|
-
if (baseline.ariaSnapshotPaths?.[i]) {
|
|
157
|
-
try {
|
|
158
|
-
baselineAria = await fs.readFile(baseline.ariaSnapshotPaths[i], 'utf-8');
|
|
159
|
-
} catch {
|
|
160
|
-
baselineAria = undefined;
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
const base: StepClassification = {
|
|
165
|
-
step: i,
|
|
166
|
-
intentional: false,
|
|
167
|
-
confidence: 0.5,
|
|
168
|
-
reasoning: 'aria snapshots unavailable for this pair — inspect the diff image manually',
|
|
169
|
-
ariaDiff: '(aria snapshots unavailable — re-record the baseline to enable classification)',
|
|
170
|
-
oldImagePath: baseline.screenshotPaths[i],
|
|
171
|
-
newImagePath: current.screenshotPaths[i],
|
|
172
|
-
};
|
|
173
|
-
|
|
174
|
-
if (baselineAria === undefined || currentAria === undefined) {
|
|
175
|
-
classifier.logger?.warn('aria snapshots unavailable for step — escalating without classification', { step: i });
|
|
176
|
-
return base;
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
const ariaDiff = diffAriaSnapshots(baselineAria, currentAria);
|
|
180
|
-
try {
|
|
181
|
-
const verdict = await classifyChange({
|
|
182
|
-
baselineAria,
|
|
183
|
-
currentAria,
|
|
184
|
-
context: classifier.context,
|
|
185
|
-
config: classifier.config,
|
|
186
|
-
apiToken: classifier.apiToken,
|
|
187
|
-
meta: {
|
|
188
|
-
flowName: baseline.metadata.name,
|
|
189
|
-
flowHash: flowFingerprint(baseline.metadata.name, baseline.steps),
|
|
190
|
-
step: i,
|
|
191
|
-
},
|
|
192
|
-
logger: classifier.logger,
|
|
193
|
-
});
|
|
194
|
-
classifier.logger?.info('classification', { step: i, ...verdict });
|
|
195
|
-
return { ...base, ...verdict, ariaDiff };
|
|
196
|
-
} catch (err) {
|
|
197
|
-
if (err instanceof FlowLimitError) {
|
|
198
|
-
classifier.logger?.warn('flow limit reached — step escalates pixel-diff only', {
|
|
199
|
-
step: i,
|
|
200
|
-
kind: err.kind,
|
|
201
|
-
flowsUsed: err.flowsUsed,
|
|
202
|
-
flowLimit: err.flowLimit,
|
|
203
|
-
});
|
|
204
|
-
const where = err.kind === 'monthly_cap_reached'
|
|
205
|
-
? 'monthly classification cap reached on the free plan'
|
|
206
|
-
: `free plan flow limit reached (${err.flowsUsed ?? '?'}/${err.flowLimit ?? '?'} flows guarded)`;
|
|
207
|
-
return {
|
|
208
|
-
...base,
|
|
209
|
-
ariaDiff,
|
|
210
|
-
reasoning: `${where} — this flow degrades to pixel-diff only and is treated as a change. Upgrade at ${err.upgradeUrl ?? 'https://kintsugi.dev/upgrade'} to guard it with the classifier.`,
|
|
211
|
-
};
|
|
212
|
-
}
|
|
213
|
-
classifier.logger?.warn('classifier failed — escalating step', { step: i, error: (err as Error).message });
|
|
214
|
-
return { ...base, ariaDiff, reasoning: `classifier failed (${(err as Error).message}) — inspect the aria diff manually` };
|
|
215
|
-
}
|
|
216
|
-
}));
|
|
217
|
-
}
|
package/src/config.test.ts
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
import { test, describe } from 'node:test';
|
|
2
|
-
import assert from 'node:assert/strict';
|
|
3
|
-
import os from 'node:os';
|
|
4
|
-
import path from 'node:path';
|
|
5
|
-
import fs from 'node:fs/promises';
|
|
6
|
-
import {
|
|
7
|
-
getDefaultConfig,
|
|
8
|
-
saveConfig,
|
|
9
|
-
loadConfig,
|
|
10
|
-
ensureKintsugiDir,
|
|
11
|
-
formatAgentFeedback,
|
|
12
|
-
formatUserSummary,
|
|
13
|
-
DiffResult,
|
|
14
|
-
type ComparisonResult
|
|
15
|
-
} from './index.js';
|
|
16
|
-
|
|
17
|
-
describe('Kintsugi Config & Feedback Formatter', () => {
|
|
18
|
-
test('generates valid default config', () => {
|
|
19
|
-
const config = getDefaultConfig();
|
|
20
|
-
assert.equal(config.version, 1);
|
|
21
|
-
assert.equal(config.viewport.width, 1280);
|
|
22
|
-
assert.equal(config.viewport.height, 720);
|
|
23
|
-
assert.ok(config.thresholds.pixelDiffPercent > 0);
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
test('saves and loads config from disk', async () => {
|
|
27
|
-
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'kintsugi-test-'));
|
|
28
|
-
try {
|
|
29
|
-
await ensureKintsugiDir(tmpDir);
|
|
30
|
-
const config = getDefaultConfig();
|
|
31
|
-
config.devServerUrl = 'http://localhost:5173';
|
|
32
|
-
await saveConfig(tmpDir, config);
|
|
33
|
-
|
|
34
|
-
const loaded = await loadConfig(tmpDir);
|
|
35
|
-
assert.equal(loaded.devServerUrl, 'http://localhost:5173');
|
|
36
|
-
} finally {
|
|
37
|
-
await fs.rm(tmpDir, { recursive: true, force: true });
|
|
38
|
-
}
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
test('formats actionable agent feedback on regression', () => {
|
|
42
|
-
const mockResult: ComparisonResult = {
|
|
43
|
-
result: DiffResult.BROKEN,
|
|
44
|
-
pixelDiffPercent: 3.42,
|
|
45
|
-
ssimScore: 1,
|
|
46
|
-
failedStep: 2,
|
|
47
|
-
stepResults: [
|
|
48
|
-
{
|
|
49
|
-
step: 2,
|
|
50
|
-
pixelDiffPercent: 3.42,
|
|
51
|
-
ssimScore: 1,
|
|
52
|
-
result: DiffResult.BROKEN,
|
|
53
|
-
}
|
|
54
|
-
],
|
|
55
|
-
classifications: [
|
|
56
|
-
{
|
|
57
|
-
step: 2,
|
|
58
|
-
intentional: false,
|
|
59
|
-
confidence: 0.9,
|
|
60
|
-
reasoning: 'The checkout button was removed; re-add it to the order summary.',
|
|
61
|
-
ariaDiff: 'REMOVED: - button "Complete Purchase ($89.00)"',
|
|
62
|
-
oldImagePath: '/baseline/step_2.png',
|
|
63
|
-
newImagePath: '/tmp/step_2.png',
|
|
64
|
-
}
|
|
65
|
-
]
|
|
66
|
-
};
|
|
67
|
-
|
|
68
|
-
const feedback = formatAgentFeedback(mockResult, 'checkout-flow');
|
|
69
|
-
assert.ok(feedback.includes('checkout-flow is broken'), 'Should state the flow is broken');
|
|
70
|
-
assert.ok(feedback.includes('aria diffs:'), 'Should include aria diffs');
|
|
71
|
-
assert.ok(feedback.includes('REMOVED: - button'), 'Should include the aria diff content');
|
|
72
|
-
assert.ok(feedback.includes('image pairs (old image | new image):'), 'Should include image pairs');
|
|
73
|
-
assert.ok(feedback.includes('/baseline/step_2.png | /tmp/step_2.png'), 'Should include both image paths');
|
|
74
|
-
assert.ok(feedback.includes('suggested fixes:'), 'Should include suggested fixes');
|
|
75
|
-
assert.ok(feedback.includes('re-add it to the order summary'), 'Should include the classifier fix');
|
|
76
|
-
|
|
77
|
-
const summary = formatUserSummary(mockResult, 'checkout-flow');
|
|
78
|
-
assert.ok(summary.includes('checkout-flow'), 'Summary should include flow name');
|
|
79
|
-
});
|
|
80
|
-
});
|
package/src/config.ts
DELETED
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
import { promises as fs } from 'fs';
|
|
2
|
-
import path from 'path';
|
|
3
|
-
import type { KintsugiConfig } from './types.js';
|
|
4
|
-
|
|
5
|
-
export function getDefaultConfig(): KintsugiConfig {
|
|
6
|
-
return {
|
|
7
|
-
version: 1,
|
|
8
|
-
devServerUrl: 'http://localhost:3000',
|
|
9
|
-
viewport: { width: 1280, height: 720 },
|
|
10
|
-
thresholds: {
|
|
11
|
-
pixelDiffPercent: 0.1,
|
|
12
|
-
ssimScore: 0.98,
|
|
13
|
-
maxCheckLatencyMs: 5000,
|
|
14
|
-
},
|
|
15
|
-
llm: {
|
|
16
|
-
provider: 'claude',
|
|
17
|
-
model: 'claude-3-5-sonnet-20241022',
|
|
18
|
-
apiKeyEnvVar: 'ANTHROPIC_API_KEY',
|
|
19
|
-
},
|
|
20
|
-
classifier: {
|
|
21
|
-
enabled: true,
|
|
22
|
-
provider: 'kintsugi',
|
|
23
|
-
endpoint: process.env.KINTSUGI_API_URL ?? 'https://api.kintsugi.dev/v1',
|
|
24
|
-
model: 'kintsugi-hosted',
|
|
25
|
-
tokenEnvVar: 'KINTSUGI_API_KEY',
|
|
26
|
-
pixelDiffThreshold: 0.5,
|
|
27
|
-
confidenceThreshold: 0.7,
|
|
28
|
-
timeoutMs: 20000,
|
|
29
|
-
},
|
|
30
|
-
agent: {
|
|
31
|
-
type: 'auto',
|
|
32
|
-
maxRetries: 3,
|
|
33
|
-
},
|
|
34
|
-
recording: {
|
|
35
|
-
passive: true,
|
|
36
|
-
collectorPort: 4321,
|
|
37
|
-
},
|
|
38
|
-
uiFilePatterns: ['src/**/*.tsx', 'src/**/*.ts', 'src/**/*.jsx', 'src/**/*.js'],
|
|
39
|
-
};
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export async function ensureKintsugiDir(projectDir: string): Promise<void> {
|
|
43
|
-
const kintsugiDir = path.join(projectDir, '.kintsugi');
|
|
44
|
-
const flowsDir = path.join(kintsugiDir, 'flows');
|
|
45
|
-
const recordingsDir = path.join(kintsugiDir, 'recordings');
|
|
46
|
-
|
|
47
|
-
await fs.mkdir(kintsugiDir, { recursive: true });
|
|
48
|
-
await fs.mkdir(flowsDir, { recursive: true });
|
|
49
|
-
await fs.mkdir(recordingsDir, { recursive: true });
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export async function loadConfig(projectDir: string): Promise<KintsugiConfig> {
|
|
53
|
-
const configPath = path.join(projectDir, '.kintsugi', 'config.json');
|
|
54
|
-
const defaultConfig = getDefaultConfig();
|
|
55
|
-
try {
|
|
56
|
-
const data = await fs.readFile(configPath, 'utf-8');
|
|
57
|
-
const userConfig = JSON.parse(data) as Partial<KintsugiConfig>;
|
|
58
|
-
const classifier = {
|
|
59
|
-
...defaultConfig.classifier,
|
|
60
|
-
...(userConfig.classifier ?? {}),
|
|
61
|
-
} as KintsugiConfig['classifier'];
|
|
62
|
-
// Migration: configs written before the provider field pointed at
|
|
63
|
-
// OpenAI-compatible endpoints (HF router). Keep them working unchanged.
|
|
64
|
-
if (userConfig.classifier && userConfig.classifier.provider === undefined) {
|
|
65
|
-
if (!classifier.endpoint.includes('kintsugi')) {
|
|
66
|
-
classifier.provider = 'openai-compatible';
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
return {
|
|
70
|
-
...defaultConfig,
|
|
71
|
-
...userConfig,
|
|
72
|
-
classifier,
|
|
73
|
-
} as KintsugiConfig;
|
|
74
|
-
} catch (error) {
|
|
75
|
-
return defaultConfig;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
export async function saveConfig(projectDir: string, config: KintsugiConfig): Promise<void> {
|
|
80
|
-
await ensureKintsugiDir(projectDir);
|
|
81
|
-
const configPath = path.join(projectDir, '.kintsugi', 'config.json');
|
|
82
|
-
await fs.writeFile(configPath, JSON.stringify(config, null, 2), 'utf-8');
|
|
83
|
-
}
|
package/src/domdiff.ts
DELETED
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Line diff for Playwright aria snapshots (YAML). Produces a compact
|
|
3
|
-
* removed/added summary suitable as classifier input.
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
const MAX_DIFF_LINES = 80;
|
|
7
|
-
|
|
8
|
-
export function diffAriaSnapshots(baseline: string, current: string): string {
|
|
9
|
-
const a = baseline.split('\n');
|
|
10
|
-
const b = current.split('\n');
|
|
11
|
-
|
|
12
|
-
// LCS table (snapshots are small — a few hundred lines max)
|
|
13
|
-
const m = a.length, n = b.length;
|
|
14
|
-
const dp: number[][] = Array.from({ length: m + 1 }, () => new Array(n + 1).fill(0));
|
|
15
|
-
for (let i = m - 1; i >= 0; i--) {
|
|
16
|
-
for (let j = n - 1; j >= 0; j--) {
|
|
17
|
-
dp[i][j] = a[i] === b[j] ? dp[i + 1][j + 1] + 1 : Math.max(dp[i + 1][j], dp[i][j + 1]);
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
// Walk the table, grouping consecutive changes into hunks with nearest context
|
|
22
|
-
type Op = { type: ' ' | '-' | '+'; line: string };
|
|
23
|
-
const ops: Op[] = [];
|
|
24
|
-
let i = 0, j = 0;
|
|
25
|
-
while (i < m && j < n) {
|
|
26
|
-
if (a[i] === b[j]) { ops.push({ type: ' ', line: a[i] }); i++; j++; }
|
|
27
|
-
else if (dp[i + 1][j] >= dp[i][j + 1]) { ops.push({ type: '-', line: a[i] }); i++; }
|
|
28
|
-
else { ops.push({ type: '+', line: b[j] }); j++; }
|
|
29
|
-
}
|
|
30
|
-
while (i < m) { ops.push({ type: '-', line: a[i++] }); }
|
|
31
|
-
while (j < n) { ops.push({ type: '+', line: b[j++] }); }
|
|
32
|
-
|
|
33
|
-
// Emit only changed hunks (with 1 line of context), trimmed.
|
|
34
|
-
// Prefixes are words, not +/-, because aria snapshot lines themselves
|
|
35
|
-
// start with YAML list markers ("- ").
|
|
36
|
-
const out: string[] = [];
|
|
37
|
-
let changes = 0;
|
|
38
|
-
let lastContextIndex = -1;
|
|
39
|
-
for (let k = 0; k < ops.length; k++) {
|
|
40
|
-
if (ops[k].type === ' ') continue;
|
|
41
|
-
if (out.length > 0 && out[out.length - 1] !== '…' && ops[k - 1]?.type === ' ' && (k - 1) !== lastContextIndex) {
|
|
42
|
-
out.push(' ctx: ' + ops[k - 1].line);
|
|
43
|
-
}
|
|
44
|
-
while (k < ops.length && ops[k].type !== ' ') {
|
|
45
|
-
out.push((ops[k].type === '-' ? 'REMOVED: ' : 'ADDED: ') + ops[k].line);
|
|
46
|
-
changes++;
|
|
47
|
-
k++;
|
|
48
|
-
}
|
|
49
|
-
if (k < ops.length) {
|
|
50
|
-
out.push(' ctx: ' + ops[k].line);
|
|
51
|
-
lastContextIndex = k;
|
|
52
|
-
}
|
|
53
|
-
out.push('');
|
|
54
|
-
if (out.length >= MAX_DIFF_LINES) {
|
|
55
|
-
out.push('… (diff truncated, more changes below)');
|
|
56
|
-
break;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
if (changes === 0) return '(no line-level differences — change is purely visual/rendered)';
|
|
61
|
-
return out.join('\n').trim();
|
|
62
|
-
}
|
package/src/feedback.ts
DELETED
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
import { DiffResult, type ComparisonResult } from './types.js';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Fix-it feedback in the agent-facing format:
|
|
5
|
-
* "{flow name} is broken, aria diffs: [...], [sets of (new image and old image)],
|
|
6
|
-
* suggested fixes: [fixes from the classifier per image pair]"
|
|
7
|
-
*/
|
|
8
|
-
export function formatAgentFeedback(result: ComparisonResult, flowName: string): string {
|
|
9
|
-
const lines: string[] = [];
|
|
10
|
-
lines.push(`${flowName} is broken.`);
|
|
11
|
-
|
|
12
|
-
// aria diffs per classified pair
|
|
13
|
-
lines.push('aria diffs:');
|
|
14
|
-
const classifications = result.classifications ?? [];
|
|
15
|
-
if (classifications.length === 0) {
|
|
16
|
-
lines.push(' (none captured — flow failed during replay; check selectors and page load)');
|
|
17
|
-
}
|
|
18
|
-
for (const c of classifications) {
|
|
19
|
-
lines.push(` step ${c.step}:`);
|
|
20
|
-
for (const line of c.ariaDiff.split('\n')) {
|
|
21
|
-
lines.push(` ${line}`);
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
// sets of (old image, new image)
|
|
26
|
-
lines.push('image pairs (old image | new image):');
|
|
27
|
-
if (classifications.length === 0) {
|
|
28
|
-
lines.push(' (none — flow failed during replay)');
|
|
29
|
-
}
|
|
30
|
-
for (const c of classifications) {
|
|
31
|
-
lines.push(` step ${c.step}: ${c.oldImagePath ?? '(unavailable)'} | ${c.newImagePath ?? '(unavailable)'}`);
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
// suggested fixes — the classifier's reasoning per non-intentional pair
|
|
35
|
-
lines.push('suggested fixes:');
|
|
36
|
-
const regressions = classifications.filter(c => !c.intentional || c.confidence <= 0.7);
|
|
37
|
-
if (classifications.length === 0) {
|
|
38
|
-
lines.push(' - re-run the flow and verify the page loads; an element selector likely changed');
|
|
39
|
-
}
|
|
40
|
-
for (const c of regressions) {
|
|
41
|
-
lines.push(` step ${c.step}: ${c.reasoning}`);
|
|
42
|
-
}
|
|
43
|
-
if (classifications.length > 0 && regressions.length === 0) {
|
|
44
|
-
lines.push(' - (classifier found no intentional-only pairs, but the flow still failed — see replay errors)');
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
if (result.diffImagePath) {
|
|
48
|
-
lines.push(`diff heatmap: ${result.diffImagePath}`);
|
|
49
|
-
}
|
|
50
|
-
return lines.join('\n');
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export function formatUserSummary(result: ComparisonResult, flowName: string): string {
|
|
54
|
-
if (result.result === DiffResult.IDENTICAL) {
|
|
55
|
-
return `✅ Flow "${flowName}" passed. No visual differences detected.`;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
let summary = `⚠️ Flow "${flowName}" resulted in ${result.result.toUpperCase()}.\n`;
|
|
59
|
-
if (result.failedStep !== undefined) {
|
|
60
|
-
summary += `Step ${result.failedStep} failed with a pixel difference of ${result.pixelDiffPercent.toFixed(2)}%.`;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
return summary;
|
|
64
|
-
}
|
package/src/flowid.ts
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Stable per-flow fingerprint used by the hosted API's flow gate: hashed from
|
|
3
|
-
* the flow name + step actions/selectors/urls — NOT from screenshots or
|
|
4
|
-
* timestamps — so replays count once and renames can't evade the limit.
|
|
5
|
-
*/
|
|
6
|
-
import { createHash } from 'node:crypto';
|
|
7
|
-
import type { FlowStep } from './types.js';
|
|
8
|
-
|
|
9
|
-
export function flowFingerprint(name: string, steps: FlowStep[]): string {
|
|
10
|
-
const canonical = JSON.stringify(
|
|
11
|
-
steps.map(s => [s.action, s.selector ?? null, s.url ?? null, s.value ?? null])
|
|
12
|
-
);
|
|
13
|
-
return createHash('sha256').update(`${name}\n${canonical}`).digest('hex').slice(0, 40);
|
|
14
|
-
}
|
package/src/index.ts
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
export * from './types.js';
|
|
2
|
-
export * from './config.js';
|
|
3
|
-
export * from './setupPrompt.js';
|
|
4
|
-
export * from './baseline.js';
|
|
5
|
-
export * from './capturer.js';
|
|
6
|
-
export * from './comparator.js';
|
|
7
|
-
export * from './feedback.js';
|
|
8
|
-
export * from './recorder.js';
|
|
9
|
-
export * from './benchmark.js';
|
|
10
|
-
export * from './classifier.js';
|
|
11
|
-
export * from './auth.js';
|
|
12
|
-
export * from './flowid.js';
|
|
13
|
-
export * from './domdiff.js';
|