@kintsugi-ai/core 0.1.0

package/src/baseline.ts

@@ -0,0 +1,125 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+import type { FlowMetadata, FlowRecording, CaptureResult } from './types.js';
+
+export function getFlowDir(projectDir: string, name: string): string {
+  return path.join(projectDir, '.kintsugi', 'flows', name);
+}
+
+export async function listFlows(projectDir: string): Promise<FlowMetadata[]> {
+  const flowsDir = path.join(projectDir, '.kintsugi', 'flows');
+  try {
+    const entries = await fs.readdir(flowsDir, { withFileTypes: true });
+    const flows: FlowMetadata[] = [];
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        const metaPath = path.join(flowsDir, entry.name, 'metadata.json');
+        try {
+          const data = await fs.readFile(metaPath, 'utf-8');
+          flows.push(JSON.parse(data) as FlowMetadata);
+        } catch (e) {
+          // Ignore invalid directories
+        }
+      }
+    }
+    return flows;
+  } catch (e) {
+    return [];
+  }
+}
+
+export async function loadFlow(projectDir: string, name: string): Promise<FlowRecording> {
+  const flowDir = getFlowDir(projectDir, name);
+  const metaPath = path.join(flowDir, 'metadata.json');
+  const stepsPath = path.join(flowDir, 'steps.json');
+  
+  const [metaData, stepsData] = await Promise.all([
+    fs.readFile(metaPath, 'utf-8'),
+    fs.readFile(stepsPath, 'utf-8'),
+  ]);
+  
+  const metadata = JSON.parse(metaData) as FlowMetadata;
+  const steps = JSON.parse(stepsData) as FlowRecording['steps'];
+  
+  const screenshotsDir = path.join(flowDir, 'screenshots');
+  let screenshotPaths: string[] = [];
+  try {
+    const files = await fs.readdir(screenshotsDir);
+    // Explicitly match step screenshots for each step in order
+    const stepFiles: string[] = [];
+    let allStepsFound = true;
+    for (let i = 0; i < steps.length; i++) {
+      const candidates = [`step_${i}.png`, `step-${String(i).padStart(3, '0')}.png`, `step-${i}.png`];
+      const match = candidates.find(c => files.includes(c));
+      if (match) {
+        stepFiles.push(path.join(screenshotsDir, match));
+      } else {
+        allStepsFound = false;
+        break;
+      }
+    }
+
+    if (allStepsFound && stepFiles.length > 0) {
+      screenshotPaths = stepFiles;
+    } else {
+      screenshotPaths = files
+        .filter((f) => f.endsWith('.png'))
+        .sort()
+        .map((f) => path.join(screenshotsDir, f));
+    }
+  } catch (e) {
+    // No screenshots directory
+  }
+
+  const videoPath = path.join(flowDir, 'video.webm');
+  const videoExists = await fs.stat(videoPath).then(() => true).catch(() => false);
+
+  // Baseline aria snapshots (semantic DOM state per step)
+  const ariaDir = path.join(flowDir, 'aria');
+  const ariaSnapshotPaths: (string | undefined)[] = [];
+  for (let i = 0; i < steps.length; i++) {
+    const ariaPath = path.join(ariaDir, `step_${i}.yml`);
+    const exists = await fs.stat(ariaPath).then(() => true).catch(() => false);
+    ariaSnapshotPaths.push(exists ? ariaPath : undefined);
+  }
+
+  return {
+    metadata,
+    steps,
+    screenshotPaths,
+    ariaSnapshotPaths: ariaSnapshotPaths.some(p => p !== undefined) ? ariaSnapshotPaths as string[] : undefined,
+    videoPath: videoExists ? videoPath : undefined,
+  };
+}
+
+export async function saveFlow(projectDir: string, recording: FlowRecording): Promise<void> {
+  const flowDir = getFlowDir(projectDir, recording.metadata.name);
+  await fs.mkdir(flowDir, { recursive: true });
+  await fs.mkdir(path.join(flowDir, 'screenshots'), { recursive: true });
+
+  await fs.writeFile(path.join(flowDir, 'metadata.json'), JSON.stringify(recording.metadata, null, 2));
+  await fs.writeFile(path.join(flowDir, 'steps.json'), JSON.stringify(recording.steps, null, 2));
+}
+
+export async function deleteFlow(projectDir: string, name: string): Promise<void> {
+  const flowDir = getFlowDir(projectDir, name);
+  await fs.rm(flowDir, { recursive: true, force: true });
+}
+
+export async function updateBaseline(projectDir: string, name: string, capture: CaptureResult): Promise<void> {
+  const flowDir = getFlowDir(projectDir, name);
+  const screenshotsDir = path.join(flowDir, 'screenshots');
+  const ariaDir = path.join(flowDir, 'aria');
+
+  await fs.mkdir(screenshotsDir, { recursive: true });
+  await fs.mkdir(ariaDir, { recursive: true });
+
+  for (let i = 0; i < capture.screenshots.length; i++) {
+    const screenshot = capture.screenshots[i];
+    await fs.writeFile(path.join(screenshotsDir, `step_${i}.png`), screenshot);
+    const aria = capture.ariaSnapshots?.[i];
+    if (aria !== undefined) {
+      await fs.writeFile(path.join(ariaDir, `step_${i}.yml`), aria, 'utf-8');
+    }
+  }
+}

package/src/benchmark.ts

@@ -0,0 +1,307 @@
+import { PNG } from 'pngjs';
+import { computePixelDiff, compareScreenshots } from './comparator.js';
+import { DiffResult } from './types.js';
+
+export interface LatencyMetric {
+  name: string;
+  targetMs: number;
+  actualMeanMs: number;
+  p50Ms: number;
+  p95Ms: number;
+  passed: boolean;
+}
+
+export interface AccuracyMetric {
+  name: string;
+  targetPercent: number;
+  actualPercent: number;
+  description: string;
+  passed: boolean;
+}
+
+export interface BenchmarkReport {
+  timestamp: string;
+  performance: {
+    metrics: LatencyMetric[];
+    allPassed: boolean;
+  };
+  accuracy: {
+    metrics: AccuracyMetric[];
+    truePositiveRate: number;
+    falsePositiveRate: number;
+    allPassed: boolean;
+  };
+  summary: {
+    totalBenchmarks: number;
+    passedBenchmarks: number;
+    launchReady: boolean;
+  };
+}
+
+/**
+ * Creates a synthetic UI PNG buffer of the given width and height.
+ */
+export function createSyntheticUiPng(options?: {
+  width?: number;
+  height?: number;
+  removeButton?: boolean;
+  shiftLayout?: boolean;
+  addNoise?: boolean;
+  noiseAmount?: number;
+}): Buffer {
+  const width = options?.width || 1280;
+  const height = options?.height || 720;
+  const png = new PNG({ width, height });
+
+  // Background - light gray #F8FAFC
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const idx = (y * width + x) * 4;
+      png.data[idx] = 248;
+      png.data[idx + 1] = 250;
+      png.data[idx + 2] = 252;
+      png.data[idx + 3] = 255;
+    }
+  }
+
+  // Draw Navbar (0 to 60px height) - #1E293B
+  for (let y = 0; y < 60; y++) {
+    for (let x = 0; x < width; x++) {
+      const idx = (y * width + x) * 4;
+      png.data[idx] = 30;
+      png.data[idx + 1] = 41;
+      png.data[idx + 2] = 59;
+    }
+  }
+
+  // Draw Card Container
+  const cardTop = options?.shiftLayout ? 120 : 100;
+  const cardLeft = 100;
+  const cardWidth = width - 200;
+  const cardHeight = 400;
+
+  for (let y = cardTop; y < cardTop + cardHeight; y++) {
+    for (let x = cardLeft; x < cardLeft + cardWidth; x++) {
+      const idx = (y * width + x) * 4;
+      png.data[idx] = 255;
+      png.data[idx + 1] = 255;
+      png.data[idx + 2] = 255;
+    }
+  }
+
+  // Draw Button (e.g. checkout button) inside card if not removed
+  if (!options?.removeButton) {
+    const btnTop = cardTop + 300;
+    const btnLeft = cardLeft + 50;
+    const btnWidth = 200;
+    const btnHeight = 48;
+
+    for (let y = btnTop; y < btnTop + btnHeight; y++) {
+      for (let x = btnLeft; x < btnLeft + btnWidth; x++) {
+        const idx = (y * width + x) * 4;
+        png.data[idx] = 37;
+        png.data[idx + 1] = 99;
+        png.data[idx + 2] = 235; // Blue #2563EB
+      }
+    }
+  }
+
+  // Add subtle subpixel/anti-aliasing noise if requested
+  if (options?.addNoise) {
+    const amount = options.noiseAmount || 0.0005; // 0.05% of pixels
+    const totalPixels = width * height;
+    const noisePixels = Math.floor(totalPixels * amount);
+
+    for (let i = 0; i < noisePixels; i++) {
+      const px = Math.floor(Math.random() * totalPixels);
+      const idx = px * 4;
+      // Slight luminance jitter (+/- 3 in color value)
+      png.data[idx] = Math.max(0, Math.min(255, png.data[idx] + (Math.random() > 0.5 ? 2 : -2)));
+      png.data[idx + 1] = Math.max(0, Math.min(255, png.data[idx + 1] + (Math.random() > 0.5 ? 2 : -2)));
+      png.data[idx + 2] = Math.max(0, Math.min(255, png.data[idx + 2] + (Math.random() > 0.5 ? 2 : -2)));
+    }
+  }
+
+  return PNG.sync.write(png);
+}
+
+function calculatePercentiles(values: number[]): { p50: number; p95: number; mean: number } {
+  const sorted = [...values].sort((a, b) => a - b);
+  const p50 = sorted[Math.floor(sorted.length * 0.5)] || 0;
+  const p95 = sorted[Math.floor(sorted.length * 0.95)] || sorted[sorted.length - 1] || 0;
+  const mean = values.reduce((sum, v) => sum + v, 0) / (values.length || 1);
+  return { p50, p95, mean };
+}
+
+/**
+ * Runs the performance latency benchmark suite.
+ */
+export async function runPerformanceBenchmark(iterations = 10): Promise<{ metrics: LatencyMetric[]; allPassed: boolean }> {
+  const baseImg = createSyntheticUiPng();
+  const minorImg = createSyntheticUiPng({ addNoise: true, noiseAmount: 0.001 });
+
+  // 1. Benchmark pixelmatch on 1280x720
+  const pixelDiffTimes: number[] = [];
+  for (let i = 0; i < iterations; i++) {
+    const t0 = performance.now();
+    await computePixelDiff(baseImg, baseImg);
+    pixelDiffTimes.push(performance.now() - t0);
+  }
+  const pixelStats = calculatePercentiles(pixelDiffTimes);
+
+  // 2. Fast-path comparison latency (identical)
+  const fastPathTimes: number[] = [];
+  for (let i = 0; i < iterations; i++) {
+    const t0 = performance.now();
+    await compareScreenshots(baseImg, baseImg);
+    fastPathTimes.push(performance.now() - t0);
+  }
+  const fastPathStats = calculatePercentiles(fastPathTimes);
+
+  // 3. Minor-band comparison latency (sub-threshold noise)
+  const tier2Times: number[] = [];
+  for (let i = 0; i < iterations; i++) {
+    const t0 = performance.now();
+    await compareScreenshots(baseImg, minorImg);
+    tier2Times.push(performance.now() - t0);
+  }
+  const tier2Stats = calculatePercentiles(tier2Times);
+
+  const metrics: LatencyMetric[] = [
+    {
+      name: 'Pixel Diff Latency (1280x720)',
+      targetMs: 50,
+      actualMeanMs: pixelStats.mean,
+      p50Ms: pixelStats.p50,
+      p95Ms: pixelStats.p95,
+      passed: pixelStats.p95 <= 75,
+    },
+    {
+      name: 'Fast-Path Hook Latency (Identical)',
+      targetMs: 50,
+      actualMeanMs: fastPathStats.mean,
+      p50Ms: fastPathStats.p50,
+      p95Ms: fastPathStats.p95,
+      passed: fastPathStats.p95 <= 75,
+    },
+    {
+      name: 'Minor-Band Hook Latency (Sub-threshold Noise)',
+      targetMs: 250,
+      actualMeanMs: tier2Stats.mean,
+      p50Ms: tier2Stats.p50,
+      p95Ms: tier2Stats.p95,
+      passed: tier2Stats.p95 <= 300,
+    },
+  ];
+
+  return {
+    metrics,
+    allPassed: metrics.every(m => m.passed),
+  };
+}
+
+/**
+ * Runs the accuracy benchmark suite checking True Positive Rate and False Positive Rate.
+ */
+export async function runAccuracyBenchmark(sampleSize = 20): Promise<{
+  metrics: AccuracyMetric[];
+  truePositiveRate: number;
+  falsePositiveRate: number;
+  allPassed: boolean;
+}> {
+  const baseImg = createSyntheticUiPng();
+
+  let truePositives = 0;
+  let falseNegatives = 0;
+  let trueNegatives = 0;
+  let falsePositives = 0;
+
+  // Test True Positive: Actual Regressions (missing button, layout shift)
+  for (let i = 0; i < sampleSize; i++) {
+    const isShift = i % 2 === 0;
+    const brokenImg = createSyntheticUiPng({
+      removeButton: !isShift,
+      shiftLayout: isShift,
+    });
+
+    const result = await compareScreenshots(baseImg, brokenImg);
+    if (result.result === DiffResult.CHANGED || result.result === DiffResult.BROKEN) {
+      truePositives++;
+    } else {
+      falseNegatives++;
+    }
+  }
+
+  // Test False Positive: Benign changes (identical or sub-pixel noise)
+  for (let i = 0; i < sampleSize; i++) {
+    const isNoise = i % 2 === 0;
+    const benignImg = isNoise
+      ? createSyntheticUiPng({ addNoise: true, noiseAmount: 0.0003 })
+      : createSyntheticUiPng();
+
+    const result = await compareScreenshots(baseImg, benignImg);
+    if (result.result === DiffResult.IDENTICAL || result.result === DiffResult.MINOR) {
+      trueNegatives++;
+    } else {
+      falsePositives++;
+    }
+  }
+
+  const tpr = (truePositives / (truePositives + falseNegatives)) * 100;
+  const fpr = (falsePositives / (falsePositives + trueNegatives)) * 100;
+
+  const metrics: AccuracyMetric[] = [
+    {
+      name: 'True Positive Rate (Catches Real Regressions)',
+      targetPercent: 95.0,
+      actualPercent: tpr,
+      description: 'Percentage of genuine visual UI defects successfully caught',
+      passed: tpr >= 95.0,
+    },
+    {
+      name: 'False Positive Rate (Filters Benign Shifts)',
+      targetPercent: 5.0,
+      actualPercent: fpr,
+      description: 'Percentage of benign subpixel/anti-aliasing changes wrongly flagged',
+      passed: fpr <= 5.0,
+    },
+    {
+      name: 'Identical Match Precision',
+      targetPercent: 100.0,
+      actualPercent: ((trueNegatives / sampleSize) * 100),
+      description: 'Accuracy when comparing completely unchanged views',
+      passed: ((trueNegatives / sampleSize) * 100) >= 95.0,
+    },
+  ];
+
+  return {
+    metrics,
+    truePositiveRate: tpr,
+    falsePositiveRate: fpr,
+    allPassed: metrics.every(m => m.passed),
+  };
+}
+
+/**
+ * Runs the complete benchmark suite (performance and accuracy).
+ */
+export async function runAllBenchmarks(): Promise<BenchmarkReport> {
+  const perf = await runPerformanceBenchmark(10);
+  const acc = await runAccuracyBenchmark(20);
+
+  const totalBenchmarks = perf.metrics.length + acc.metrics.length;
+  const passedBenchmarks =
+    perf.metrics.filter(m => m.passed).length +
+    acc.metrics.filter(m => m.passed).length;
+
+  return {
+    timestamp: new Date().toISOString(),
+    performance: perf,
+    accuracy: acc,
+    summary: {
+      totalBenchmarks,
+      passedBenchmarks,
+      launchReady: perf.allPassed && acc.allPassed,
+    },
+  };
+}

package/src/capturer.ts

@@ -0,0 +1,105 @@
+import { chromium, type Browser } from 'playwright';
+import { promises as fs } from 'fs';
+import path from 'path';
+import type { CaptureResult, FlowRecording } from './types.js';
+
+let browserInstance: Browser | null = null;
+
+export async function getBrowser(): Promise<Browser> {
+  if (!browserInstance) {
+    browserInstance = await chromium.launch({ headless: true });
+  }
+  return browserInstance;
+}
+
+export async function closeBrowser(): Promise<void> {
+  if (browserInstance) {
+    await browserInstance.close();
+    browserInstance = null;
+  }
+}
+
+export async function captureCurrentState(options: {
+  url: string;
+  flow: FlowRecording;
+  outputDir: string;
+  viewport?: { width: number; height: number };
+}): Promise<CaptureResult> {
+  const { url, flow, outputDir, viewport = { width: 1280, height: 720 } } = options;
+  const browser = await getBrowser();
+  
+  const videoDir = path.join(outputDir, 'video');
+  await fs.mkdir(videoDir, { recursive: true });
+  
+  const context = await browser.newContext({
+    viewport,
+    recordVideo: { dir: videoDir }
+  });
+  
+  const page = await context.newPage();
+  const result: CaptureResult = {
+    screenshots: [],
+    screenshotPaths: [],
+    ariaSnapshots: [],
+    completedSteps: 0,
+    totalSteps: flow.steps.length,
+    errors: []
+  };
+
+  await fs.mkdir(outputDir, { recursive: true });
+
+  try {
+    let navigated = false;
+    for (let i = 0; i < flow.steps.length; i++) {
+      const step = flow.steps[i];
+      try {
+        if (!navigated && page.url() === 'about:blank' && step.action !== 'navigate') {
+          await page.goto(url, { waitUntil: 'networkidle' });
+          navigated = true;
+        }
+
+        if (step.action === 'navigate') {
+          await page.goto(step.url || url, { waitUntil: 'networkidle' });
+          navigated = true;
+        } else if (step.action === 'click' && step.selector) {
+          await page.click(step.selector, { timeout: step.timeout || 5000 });
+        } else if (step.action === 'type' && step.selector && step.value !== undefined) {
+          await page.fill(step.selector, step.value, { timeout: step.timeout || 5000 });
+        } else if (step.action === 'scroll' && step.position) {
+          await page.mouse.wheel(step.position.x, step.position.y);
+        } else if (step.action === 'wait') {
+          await page.waitForTimeout(step.timeout || 1000);
+        }
+
+        await page.waitForTimeout(500);
+        
+        const screenshotPath = path.join(outputDir, `step_${i}.png`);
+        const buffer = await page.screenshot({ path: screenshotPath });
+
+        // Semantic DOM state for the classifier tier (diffable as text)
+        let aria = '';
+        try {
+          aria = await page.locator('body').ariaSnapshot();
+        } catch {
+          aria = '';
+        }
+        await fs.writeFile(path.join(outputDir, `aria_step_${i}.yml`), aria, 'utf-8');
+
+        result.screenshots.push(buffer);
+        result.screenshotPaths.push(screenshotPath);
+        result.ariaSnapshots.push(aria);
+        result.completedSteps++;
+      } catch (err: any) {
+        result.errors.push(`Step ${i} (${step.action}) failed: ${err.message}`);
+        break;
+      }
+    }
+  } finally {
+    const videoObj = await page.video();
+    const videoPath = videoObj ? await videoObj.path() : undefined;
+    result.videoPath = videoPath;
+    await context.close();
+  }
+
+  return result;
+}