@kintsugi-ai/core 0.1.0

package/src/classifier.ts

@@ -0,0 +1,258 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { diffAriaSnapshots } from './domdiff.js';
+import type { ClassifierConfig } from './types.js';
+
+export interface ClassifierVerdict {
+  intentional: boolean;
+  /** 0..1 certainty reported by the classifier */
+  confidence: number;
+  /** One sentence: what changed and — if not intentional — how to fix it */
+  reasoning: string;
+}
+
+/** Metadata about the flow being classified (the hosted API uses it for the flow gate + usage log) */
+export interface ClassifyMeta {
+  flowName?: string;
+  flowHash?: string;
+  step?: number;
+}
+
+/**
+ * Thrown when the hosted API returns 429 — either the free-plan flow limit
+ * (distinct flows) or the monthly abuse cap. Carries the payload surfaced
+ * in agent feedback and UI ("10 of 10 flows guarded — upgrade for unlimited").
+ */
+export class FlowLimitError extends Error {
+  readonly kind: 'flow_limit_reached' | 'monthly_cap_reached';
+  readonly flowsUsed?: number;
+  readonly flowLimit?: number;
+  readonly upgradeUrl?: string;
+
+  constructor(body: { error?: string; flowsUsed?: number; flowLimit?: number; upgradeUrl?: string }) {
+    super(body.error ?? 'limit_reached');
+    this.name = 'FlowLimitError';
+    this.kind = body.error === 'monthly_cap_reached' ? 'monthly_cap_reached' : 'flow_limit_reached';
+    this.flowsUsed = body.flowsUsed;
+    this.flowLimit = body.flowLimit;
+    this.upgradeUrl = body.upgradeUrl;
+  }
+}
+
+const SYSTEM_PROMPT = [
+  'You are a strict UI regression classifier for an automated QA tool that guards AI coding agents.',
+  'You will be given: (1) a structural diff of the page accessibility tree between the accepted baseline and the current state after a code change, and (2) optional context about what the developer/agent was asked to do in the latest turn.',
+  'The baseline is the ACCEPTED, WORKING state. Removed or broken elements (buttons, forms, prices, text, navigation) are regressions even if the remaining page looks coherent.',
+  'If no context is provided, be conservative: significant structural changes should be classified as not intentional.',
+  'If context IS provided and the diff implements exactly what the context asked for, classify it as intentional with high confidence — a requested change is not a regression.',
+  'Benign changes (reordered identical items, minor text edits, added requested features) are intentional.',
+  'Answer with ONLY a JSON object, no other text:',
+  '{"intentional": <true|false>, "confidence": <number between 0 and 1>, "reasoning": "<exactly one sentence: what changed, and if not intentional, the specific fix>"}',
+].join('\n');
+
+/**
+ * Classifies a visual change as intentional or a regression. Provider
+ * 'kintsugi' POSTs to the hosted classifier API (which owns the model and
+ * the flow gate); 'openai-compatible' sends the chat-completions prompt
+ * below to any OpenAI-compatible endpoint.
+ */
+export async function classifyChange(options: {
+  baselineAria: string;
+  currentAria: string;
+  context?: string;
+  config: ClassifierConfig;
+  apiToken: string;
+  meta?: ClassifyMeta;
+  logger?: { info: (msg: string, data?: unknown) => void; warn: (msg: string, data?: unknown) => void };
+}): Promise<ClassifierVerdict> {
+  if (options.config.provider === 'kintsugi') {
+    return classifyViaKintsugi(options);
+  }
+  return classifyViaOpenAiCompatible(options);
+}
+
+/** Hosted kintsugi API path — the server computes the diff, picks the model, and enforces the flow gate. */
+async function classifyViaKintsugi(options: {
+  baselineAria: string;
+  currentAria: string;
+  context?: string;
+  config: ClassifierConfig;
+  apiToken: string;
+  meta?: ClassifyMeta;
+}): Promise<ClassifierVerdict> {
+  const { baselineAria, currentAria, context, config, apiToken, meta } = options;
+  const url = `${config.endpoint.replace(/\/$/, '')}/classify`;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), config.timeoutMs);
+  let response: Response;
+  try {
+    response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${apiToken}`,
+      },
+      body: JSON.stringify({ baselineAria, currentAria, context: context ?? null, meta: meta ?? null }),
+      signal: controller.signal,
+    });
+  } finally {
+    clearTimeout(timer);
+  }
+
+  if (response.status === 429) {
+    const body = await response.json().catch(() => ({}));
+    throw new FlowLimitError(body);
+  }
+  if (!response.ok) {
+    const detail = await response.text().catch(() => '');
+    throw new Error(`kintsugi api ${response.status}: ${detail.slice(0, 300)}`);
+  }
+  const payload = (await response.json()) as Partial<ClassifierVerdict>;
+  if (typeof payload.intentional !== 'boolean' || typeof payload.confidence !== 'number') {
+    throw new Error(`unexpected kintsugi api response: ${JSON.stringify(payload).slice(0, 200)}`);
+  }
+  return {
+    intentional: payload.intentional,
+    confidence: payload.confidence,
+    reasoning: payload.reasoning ?? '',
+  };
+}
+
+async function classifyViaOpenAiCompatible(options: {
+  baselineAria: string;
+  currentAria: string;
+  context?: string;
+  config: ClassifierConfig;
+  apiToken: string;
+  logger?: { info: (msg: string, data?: unknown) => void; warn: (msg: string, data?: unknown) => void };
+}): Promise<ClassifierVerdict> {
+  const { baselineAria, currentAria, context, config, apiToken, logger } = options;
+
+  const diff = diffAriaSnapshots(baselineAria, currentAria);
+  const userText = [
+    'Accessibility-tree diff (baseline → current). Lines prefixed "REMOVED:" were removed from the page, "ADDED:" were added:',
+    diff,
+    '',
+    context
+      ? `Context from the latest turn (what the agent was asked to do):\n"${context}"`
+      : 'No context available about the intended change.',
+    '',
+    'Classify this change as intentional or not. Reply with ONLY the JSON object.',
+  ].join('\n');
+
+  const body = {
+    model: config.model,
+    temperature: 0,
+    max_tokens: 120,
+    messages: [
+      { role: 'system', content: SYSTEM_PROMPT },
+      { role: 'user', content: userText },
+    ],
+  };
+
+  const url = `${config.endpoint.replace(/\/$/, '')}/chat/completions`;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), config.timeoutMs);
+  let response: Response;
+  try {
+    response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${apiToken}`,
+      },
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    });
+  } finally {
+    clearTimeout(timer);
+  }
+
+  if (!response.ok) {
+    const detail = await response.text().catch(() => '');
+    throw new Error(`classifier endpoint ${response.status}: ${detail.slice(0, 300)}`);
+  }
+
+  const payload = JSON.parse(await response.text()) as {
+    choices?: { message?: { content?: string } }[];
+  };
+  const content = payload.choices?.[0]?.message?.content ?? '';
+  logger?.info('classifier responded', { content: content.slice(0, 300) });
+  return parseVerdict(content);
+}
+
+/** Robustly extracts {intentional, confidence, reasoning} from the model output. Fails safe to not-intentional. */
+export function parseVerdict(content: string): ClassifierVerdict {
+  const text = content.trim();
+
+  // Preferred: the JSON object we asked for
+  const jsonMatch = text.match(/\{[\s\S]*?\}/);
+  if (jsonMatch) {
+    try {
+      const parsed = JSON.parse(jsonMatch[0]) as {
+        intentional?: boolean;
+        confidence?: number;
+        reasoning?: string;
+      };
+      if (typeof parsed.intentional === 'boolean') {
+        return {
+          intentional: parsed.intentional,
+          confidence: typeof parsed.confidence === 'number'
+            ? Math.min(1, Math.max(0, parsed.confidence))
+            : 0.5,
+          reasoning: typeof parsed.reasoning === 'string' && parsed.reasoning.trim()
+            ? parsed.reasoning.trim()
+            : 'No reasoning provided by classifier.',
+        };
+      }
+    } catch {
+      // fall through to fail-safe
+    }
+  }
+
+  // Unparseable — fail safe: treat as an unresolved regression
+  return {
+    intentional: false,
+    confidence: 0.5,
+    reasoning: 'Classifier output was unparseable; treat this change as a regression and inspect the aria diff manually.',
+  };
+}
+
+/**
+ * Resolves the API token: process env first, then the project's
+ * `<projectDir>/.kintsugi/.env`, then the machine-global `~/.kintsugi/.env`
+ * (so one key works across all projects).
+ */
+export function resolveApiToken(tokenEnvVar: string, projectDir?: string): string | undefined {
+  if (process.env[tokenEnvVar]) return process.env[tokenEnvVar];
+  if (projectDir) loadKintsugiEnv(projectDir);
+  loadEnvFile(globalKintsugiEnvPath());
+  return process.env[tokenEnvVar];
+}
+
+/** Path of the machine-global kintsugi env file — where the hosted API key lives. */
+export function globalKintsugiEnvPath(): string {
+  return path.join(os.homedir(), '.kintsugi', '.env');
+}
+
+/** Loads KEY=VALUE pairs from <projectDir>/.kintsugi/.env into process.env (never overrides). */
+export function loadKintsugiEnv(projectDir: string): void {
+  loadEnvFile(path.join(projectDir, '.kintsugi', '.env'));
+}
+
+function loadEnvFile(envPath: string): void {
+  let content: string;
+  try {
+    content = fs.readFileSync(envPath, 'utf-8');
+  } catch {
+    return;
+  }
+  for (const line of content.split('\n')) {
+    const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/);
+    if (!match) continue;
+    const key = match[1];
+    const value = match[2].replace(/^["']|["']$/g, '');
+    if (!(key in process.env)) process.env[key] = value;
+  }
+}

package/src/comparator.test.ts

@@ -0,0 +1,97 @@
+import { test, describe } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  createSyntheticUiPng,
+  computePixelDiff,
+  compareScreenshots,
+  diffAriaSnapshots,
+  parseVerdict,
+  DiffResult
+} from './index.js';
+
+describe('Kintsugi Visual Comparator Engine', () => {
+  test('identical images return 0% diff and IDENTICAL status', async () => {
+    const img1 = createSyntheticUiPng();
+    const img2 = createSyntheticUiPng();
+
+    const pixelResult = await computePixelDiff(img1, img2);
+    assert.equal(pixelResult.diffPercent, 0);
+
+    const comp = await compareScreenshots(img1, img2);
+    assert.equal(comp.result, DiffResult.IDENTICAL);
+    assert.equal(comp.pixelDiffPercent, 0);
+  });
+
+  test('large pixel diff without a classifier escalates as CHANGED', async () => {
+    const baseline = createSyntheticUiPng({ removeButton: false });
+    const regressed = createSyntheticUiPng({ removeButton: true });
+
+    const pixelResult = await computePixelDiff(baseline, regressed);
+    assert.ok(pixelResult.diffPercent > 0.5, 'Pixel diff should exceed 0.5%');
+
+    const comp = await compareScreenshots(baseline, regressed);
+    assert.equal(comp.result, DiffResult.CHANGED, 'Missing button should escalate as CHANGED');
+  });
+
+  test('sub-threshold noise is classified as MINOR', async () => {
+    const baseline = createSyntheticUiPng();
+    const withNoise = createSyntheticUiPng({ addNoise: true, noiseAmount: 0.0002 });
+
+    const comp = await compareScreenshots(baseline, withNoise);
+    assert.ok(
+      comp.result === DiffResult.IDENTICAL || comp.result === DiffResult.MINOR,
+      'Subpixel noise should be classified as IDENTICAL or MINOR, not BROKEN'
+    );
+  });
+
+  test('aria snapshot diff reports removed and added lines', () => {
+    const baseline = [
+      '- button "Complete Purchase ($89.00)"',
+      '- text "Free shipping"',
+    ].join('\n');
+    const current = [
+      '- text "Free shipping"',
+      '- textbox "Email"',
+    ].join('\n');
+
+    const diff = diffAriaSnapshots(baseline, current);
+    assert.ok(diff.includes('REMOVED: - button "Complete Purchase ($89.00)"'), 'Removed button should appear as removed');
+    assert.ok(diff.includes('textbox "Email"') && diff.includes('ADDED:'), 'Added textbox should appear as added');
+    assert.ok(!diff.includes('no line-level differences'));
+  });
+
+  test('aria snapshot diff of identical snapshots reports no differences', () => {
+    const snap = '- button "Buy"\n- text "Hello"';
+    assert.ok(diffAriaSnapshots(snap, snap).includes('no line-level differences'));
+  });
+});
+
+describe('Classifier verdict parsing', () => {
+  test('parses the JSON verdict format', () => {
+    const v = parseVerdict('{"intentional": true, "confidence": 0.92, "reasoning": "Matches the requested recolor."}');
+    assert.equal(v.intentional, true);
+    assert.equal(v.confidence, 0.92);
+    assert.ok(v.reasoning.includes('recolor'));
+  });
+
+  test('parses JSON embedded in chatter', () => {
+    const v = parseVerdict('Sure! {"intentional": false, "confidence": 0.85, "reasoning": "The checkout button was removed; re-add it."} hope that helps');
+    assert.equal(v.intentional, false);
+    assert.equal(v.confidence, 0.85);
+    assert.ok(v.reasoning.includes('re-add'));
+  });
+
+  test('defaults missing confidence and reasoning conservatively', () => {
+    const v = parseVerdict('{"intentional": true}');
+    assert.equal(v.intentional, true);
+    assert.equal(v.confidence, 0.5);
+    assert.ok(v.reasoning.length > 0);
+  });
+
+  test('fails safe to not-intentional on unparseable output', () => {
+    const v = parseVerdict('I cannot tell what changed here, sorry.');
+    assert.equal(v.intentional, false);
+    assert.ok(v.confidence <= 0.7);
+    assert.ok(v.reasoning.length > 0);
+  });
+});

package/src/comparator.ts

@@ -0,0 +1,217 @@
+import pixelmatch from 'pixelmatch';
+import { PNG } from 'pngjs';
+import { promises as fs } from 'fs';
+import path from 'path';
+import { DiffResult, type StepComparisonResult, type StepClassification, type FlowRecording, type CaptureResult, type ComparisonResult, type ClassifierConfig } from './types.js';
+import { classifyChange, FlowLimitError } from './classifier.js';
+import { flowFingerprint } from './flowid.js';
+import { diffAriaSnapshots } from './domdiff.js';
+
+/** Everything the comparator needs to consult the classifier for large diffs */
+export interface ClassifierContext {
+  config: ClassifierConfig;
+  apiToken: string;
+  /** What the agent was asked to do in the latest turn (sharpens intentional-vs-regression) */
+  context?: string;
+  logger?: { info: (msg: string, data?: unknown) => void; warn: (msg: string, data?: unknown) => void };
+}
+
+export interface CompareOptions {
+  outputDir?: string;
+  classifier?: ClassifierContext;
+}
+
+export async function computePixelDiff(img1: Buffer, img2: Buffer): Promise<{ diffPercent: number; diffImage: Buffer }> {
+  const png1 = PNG.sync.read(img1);
+  const png2 = PNG.sync.read(img2);
+  
+  if (png1.width !== png2.width || png1.height !== png2.height) {
+    return { diffPercent: 100, diffImage: Buffer.alloc(0) };
+  }
+
+  const { width, height } = png1;
+  const diff = new PNG({ width, height });
+
+  const numDiffPixels = pixelmatch(png1.data, png2.data, diff.data, width, height, { threshold: 0.1 });
+  const diffPercent = (numDiffPixels / (width * height)) * 100;
+  const diffImage = PNG.sync.write(diff);
+
+  return { diffPercent, diffImage };
+}
+
+/**
+ * Pixel-only verdict for one screenshot pair. Diff percentages above the
+ * classifier threshold are returned as a tentative CHANGED — compareFlow
+ * refines those with the classifier when one is configured.
+ */
+export async function compareScreenshots(baseline: Buffer, current: Buffer, stepIndex: number = 0, options?: CompareOptions): Promise<StepComparisonResult> {
+  const outputDir = options?.outputDir;
+  const { diffPercent, diffImage } = await computePixelDiff(baseline, current);
+
+  let result: DiffResult;
+  if (diffPercent <= 0.05) {
+    result = DiffResult.IDENTICAL;
+  } else if (diffPercent > (options?.classifier?.config.pixelDiffThreshold ?? 0.5)) {
+    result = DiffResult.CHANGED;
+  } else {
+    // Sub-threshold change: benign (anti-aliasing, subpixel shifts, dynamic content)
+    result = DiffResult.MINOR;
+  }
+
+  let diffImagePath;
+  if (outputDir && result !== DiffResult.IDENTICAL) {
+    diffImagePath = path.join(outputDir, `diff_${stepIndex}.png`);
+    await fs.writeFile(diffImagePath, diffImage);
+  }
+
+  return {
+    step: stepIndex,
+    pixelDiffPercent: diffPercent,
+    ssimScore: 1, // deprecated field, kept for output compatibility
+    result,
+    diffImagePath,
+  };
+}
+
+export async function compareFlow(baseline: FlowRecording, current: CaptureResult, options?: CompareOptions): Promise<ComparisonResult> {
+  const stepResults: StepComparisonResult[] = [];
+  const len = Math.min(baseline.screenshotPaths.length, current.screenshots.length);
+
+  // Pass 1: pixel diff every step (cheap, deterministic)
+  for (let i = 0; i < len; i++) {
+    const baselineBuffer = await fs.readFile(baseline.screenshotPaths[i]);
+    const currentBuffer = current.screenshots[i];
+    stepResults.push(await compareScreenshots(baselineBuffer, currentBuffer, i, options));
+  }
+
+  // Pass 2: classify every oversized step — all classifier calls in parallel
+  const threshold = options?.classifier?.config.pixelDiffThreshold ?? 0.5;
+  const candidates = stepResults.filter(s => s.pixelDiffPercent > threshold);
+  const classifications = options?.classifier
+    ? await classifySteps(candidates, baseline, current, options.classifier)
+    : [];
+  for (const c of classifications) {
+    const step = stepResults.find(s => s.step === c.step);
+    if (step) {
+      step.result = c.intentional && c.confidence > (options!.classifier!.config.confidenceThreshold)
+        ? DiffResult.INTENTIONAL
+        : DiffResult.CHANGED;
+    }
+  }
+
+  // Aggregate the worst verdict across steps
+  let worstResult = DiffResult.IDENTICAL;
+  let failedStep: number | undefined;
+  for (const comp of stepResults) {
+    if (comp.result === DiffResult.CHANGED || comp.result === DiffResult.BROKEN || comp.result === DiffResult.ERROR) {
+      if (worstResult !== DiffResult.CHANGED && worstResult !== DiffResult.BROKEN && worstResult !== DiffResult.ERROR) {
+        worstResult = comp.result;
+        failedStep = comp.step;
+      }
+    } else if (comp.result === DiffResult.MINOR && worstResult === DiffResult.IDENTICAL) {
+      worstResult = DiffResult.MINOR;
+    } else if (comp.result === DiffResult.INTENTIONAL && worstResult === DiffResult.IDENTICAL) {
+      worstResult = DiffResult.INTENTIONAL;
+    }
+  }
+
+  if (current.errors.length > 0) {
+    worstResult = DiffResult.BROKEN;
+    failedStep = current.completedSteps;
+  }
+
+  const worstStep = stepResults.find(s => s.step === failedStep) || stepResults[0];
+
+  // Surface the classifier's explanation in the escalation panel (semanticAnalysis
+  // is what the VS Code split-view displays).
+  const worstRegression = classifications.find(c => !c.intentional);
+
+  return {
+    result: worstResult,
+    pixelDiffPercent: worstStep?.pixelDiffPercent || 0,
+    ssimScore: worstStep?.ssimScore || 1,
+    diffImagePath: worstStep?.diffImagePath,
+    semanticAnalysis: worstRegression?.reasoning,
+    failedStep,
+    stepResults,
+    classifications: classifications.length > 0 ? classifications : undefined,
+  };
+}
+
+/**
+ * Classifies all oversized steps in parallel. Fail-safe: a missing aria
+ * snapshot or any classifier error yields a not-intentional classification
+ * so the step still escalates.
+ */
+async function classifySteps(
+  candidates: StepComparisonResult[],
+  baseline: FlowRecording,
+  current: CaptureResult,
+  classifier: ClassifierContext
+): Promise<StepClassification[]> {
+  return Promise.all(candidates.map(async (candidate): Promise<StepClassification> => {
+    const i = candidate.step;
+    const currentAria = current.ariaSnapshots?.[i];
+    let baselineAria: string | undefined;
+    if (baseline.ariaSnapshotPaths?.[i]) {
+      try {
+        baselineAria = await fs.readFile(baseline.ariaSnapshotPaths[i], 'utf-8');
+      } catch {
+        baselineAria = undefined;
+      }
+    }
+
+    const base: StepClassification = {
+      step: i,
+      intentional: false,
+      confidence: 0.5,
+      reasoning: 'aria snapshots unavailable for this pair — inspect the diff image manually',
+      ariaDiff: '(aria snapshots unavailable — re-record the baseline to enable classification)',
+      oldImagePath: baseline.screenshotPaths[i],
+      newImagePath: current.screenshotPaths[i],
+    };
+
+    if (baselineAria === undefined || currentAria === undefined) {
+      classifier.logger?.warn('aria snapshots unavailable for step — escalating without classification', { step: i });
+      return base;
+    }
+
+    const ariaDiff = diffAriaSnapshots(baselineAria, currentAria);
+    try {
+      const verdict = await classifyChange({
+        baselineAria,
+        currentAria,
+        context: classifier.context,
+        config: classifier.config,
+        apiToken: classifier.apiToken,
+        meta: {
+          flowName: baseline.metadata.name,
+          flowHash: flowFingerprint(baseline.metadata.name, baseline.steps),
+          step: i,
+        },
+        logger: classifier.logger,
+      });
+      classifier.logger?.info('classification', { step: i, ...verdict });
+      return { ...base, ...verdict, ariaDiff };
+    } catch (err) {
+      if (err instanceof FlowLimitError) {
+        classifier.logger?.warn('flow limit reached — step escalates pixel-diff only', {
+          step: i,
+          kind: err.kind,
+          flowsUsed: err.flowsUsed,
+          flowLimit: err.flowLimit,
+        });
+        const where = err.kind === 'monthly_cap_reached'
+          ? 'monthly classification cap reached on the free plan'
+          : `free plan flow limit reached (${err.flowsUsed ?? '?'}/${err.flowLimit ?? '?'} flows guarded)`;
+        return {
+          ...base,
+          ariaDiff,
+          reasoning: `${where} — this flow degrades to pixel-diff only and is treated as a change. Upgrade at ${err.upgradeUrl ?? 'https://kintsugi.dev/upgrade'} to guard it with the classifier.`,
+        };
+      }
+      classifier.logger?.warn('classifier failed — escalating step', { step: i, error: (err as Error).message });
+      return { ...base, ariaDiff, reasoning: `classifier failed (${(err as Error).message}) — inspect the aria diff manually` };
+    }
+  }));
+}

package/src/config.test.ts

@@ -0,0 +1,80 @@
+import { test, describe } from 'node:test';
+import assert from 'node:assert/strict';
+import os from 'node:os';
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import {
+  getDefaultConfig,
+  saveConfig,
+  loadConfig,
+  ensureKintsugiDir,
+  formatAgentFeedback,
+  formatUserSummary,
+  DiffResult,
+  type ComparisonResult
+} from './index.js';
+
+describe('Kintsugi Config & Feedback Formatter', () => {
+  test('generates valid default config', () => {
+    const config = getDefaultConfig();
+    assert.equal(config.version, 1);
+    assert.equal(config.viewport.width, 1280);
+    assert.equal(config.viewport.height, 720);
+    assert.ok(config.thresholds.pixelDiffPercent > 0);
+  });
+
+  test('saves and loads config from disk', async () => {
+    const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'kintsugi-test-'));
+    try {
+      await ensureKintsugiDir(tmpDir);
+      const config = getDefaultConfig();
+      config.devServerUrl = 'http://localhost:5173';
+      await saveConfig(tmpDir, config);
+
+      const loaded = await loadConfig(tmpDir);
+      assert.equal(loaded.devServerUrl, 'http://localhost:5173');
+    } finally {
+      await fs.rm(tmpDir, { recursive: true, force: true });
+    }
+  });
+
+  test('formats actionable agent feedback on regression', () => {
+    const mockResult: ComparisonResult = {
+      result: DiffResult.BROKEN,
+      pixelDiffPercent: 3.42,
+      ssimScore: 1,
+      failedStep: 2,
+      stepResults: [
+        {
+          step: 2,
+          pixelDiffPercent: 3.42,
+          ssimScore: 1,
+          result: DiffResult.BROKEN,
+        }
+      ],
+      classifications: [
+        {
+          step: 2,
+          intentional: false,
+          confidence: 0.9,
+          reasoning: 'The checkout button was removed; re-add it to the order summary.',
+          ariaDiff: 'REMOVED: - button "Complete Purchase ($89.00)"',
+          oldImagePath: '/baseline/step_2.png',
+          newImagePath: '/tmp/step_2.png',
+        }
+      ]
+    };
+
+    const feedback = formatAgentFeedback(mockResult, 'checkout-flow');
+    assert.ok(feedback.includes('checkout-flow is broken'), 'Should state the flow is broken');
+    assert.ok(feedback.includes('aria diffs:'), 'Should include aria diffs');
+    assert.ok(feedback.includes('REMOVED: - button'), 'Should include the aria diff content');
+    assert.ok(feedback.includes('image pairs (old image | new image):'), 'Should include image pairs');
+    assert.ok(feedback.includes('/baseline/step_2.png | /tmp/step_2.png'), 'Should include both image paths');
+    assert.ok(feedback.includes('suggested fixes:'), 'Should include suggested fixes');
+    assert.ok(feedback.includes('re-add it to the order summary'), 'Should include the classifier fix');
+
+    const summary = formatUserSummary(mockResult, 'checkout-flow');
+    assert.ok(summary.includes('checkout-flow'), 'Summary should include flow name');
+  });
+});