@kintsugi-ai/core 0.1.0 → 0.2.0

package/src/benchmark.ts

@@ -1,307 +0,0 @@
-import { PNG } from 'pngjs';
-import { computePixelDiff, compareScreenshots } from './comparator.js';
-import { DiffResult } from './types.js';
-
-export interface LatencyMetric {
-  name: string;
-  targetMs: number;
-  actualMeanMs: number;
-  p50Ms: number;
-  p95Ms: number;
-  passed: boolean;
-}
-
-export interface AccuracyMetric {
-  name: string;
-  targetPercent: number;
-  actualPercent: number;
-  description: string;
-  passed: boolean;
-}
-
-export interface BenchmarkReport {
-  timestamp: string;
-  performance: {
-    metrics: LatencyMetric[];
-    allPassed: boolean;
-  };
-  accuracy: {
-    metrics: AccuracyMetric[];
-    truePositiveRate: number;
-    falsePositiveRate: number;
-    allPassed: boolean;
-  };
-  summary: {
-    totalBenchmarks: number;
-    passedBenchmarks: number;
-    launchReady: boolean;
-  };
-}
-
-/**
- * Creates a synthetic UI PNG buffer of the given width and height.
- */
-export function createSyntheticUiPng(options?: {
-  width?: number;
-  height?: number;
-  removeButton?: boolean;
-  shiftLayout?: boolean;
-  addNoise?: boolean;
-  noiseAmount?: number;
-}): Buffer {
-  const width = options?.width || 1280;
-  const height = options?.height || 720;
-  const png = new PNG({ width, height });
-
-  // Background - light gray #F8FAFC
-  for (let y = 0; y < height; y++) {
-    for (let x = 0; x < width; x++) {
-      const idx = (y * width + x) * 4;
-      png.data[idx] = 248;
-      png.data[idx + 1] = 250;
-      png.data[idx + 2] = 252;
-      png.data[idx + 3] = 255;
-    }
-  }
-
-  // Draw Navbar (0 to 60px height) - #1E293B
-  for (let y = 0; y < 60; y++) {
-    for (let x = 0; x < width; x++) {
-      const idx = (y * width + x) * 4;
-      png.data[idx] = 30;
-      png.data[idx + 1] = 41;
-      png.data[idx + 2] = 59;
-    }
-  }
-
-  // Draw Card Container
-  const cardTop = options?.shiftLayout ? 120 : 100;
-  const cardLeft = 100;
-  const cardWidth = width - 200;
-  const cardHeight = 400;
-
-  for (let y = cardTop; y < cardTop + cardHeight; y++) {
-    for (let x = cardLeft; x < cardLeft + cardWidth; x++) {
-      const idx = (y * width + x) * 4;
-      png.data[idx] = 255;
-      png.data[idx + 1] = 255;
-      png.data[idx + 2] = 255;
-    }
-  }
-
-  // Draw Button (e.g. checkout button) inside card if not removed
-  if (!options?.removeButton) {
-    const btnTop = cardTop + 300;
-    const btnLeft = cardLeft + 50;
-    const btnWidth = 200;
-    const btnHeight = 48;
-
-    for (let y = btnTop; y < btnTop + btnHeight; y++) {
-      for (let x = btnLeft; x < btnLeft + btnWidth; x++) {
-        const idx = (y * width + x) * 4;
-        png.data[idx] = 37;
-        png.data[idx + 1] = 99;
-        png.data[idx + 2] = 235; // Blue #2563EB
-      }
-    }
-  }
-
-  // Add subtle subpixel/anti-aliasing noise if requested
-  if (options?.addNoise) {
-    const amount = options.noiseAmount || 0.0005; // 0.05% of pixels
-    const totalPixels = width * height;
-    const noisePixels = Math.floor(totalPixels * amount);
-
-    for (let i = 0; i < noisePixels; i++) {
-      const px = Math.floor(Math.random() * totalPixels);
-      const idx = px * 4;
-      // Slight luminance jitter (+/- 3 in color value)
-      png.data[idx] = Math.max(0, Math.min(255, png.data[idx] + (Math.random() > 0.5 ? 2 : -2)));
-      png.data[idx + 1] = Math.max(0, Math.min(255, png.data[idx + 1] + (Math.random() > 0.5 ? 2 : -2)));
-      png.data[idx + 2] = Math.max(0, Math.min(255, png.data[idx + 2] + (Math.random() > 0.5 ? 2 : -2)));
-    }
-  }
-
-  return PNG.sync.write(png);
-}
-
-function calculatePercentiles(values: number[]): { p50: number; p95: number; mean: number } {
-  const sorted = [...values].sort((a, b) => a - b);
-  const p50 = sorted[Math.floor(sorted.length * 0.5)] || 0;
-  const p95 = sorted[Math.floor(sorted.length * 0.95)] || sorted[sorted.length - 1] || 0;
-  const mean = values.reduce((sum, v) => sum + v, 0) / (values.length || 1);
-  return { p50, p95, mean };
-}
-
-/**
- * Runs the performance latency benchmark suite.
- */
-export async function runPerformanceBenchmark(iterations = 10): Promise<{ metrics: LatencyMetric[]; allPassed: boolean }> {
-  const baseImg = createSyntheticUiPng();
-  const minorImg = createSyntheticUiPng({ addNoise: true, noiseAmount: 0.001 });
-
-  // 1. Benchmark pixelmatch on 1280x720
-  const pixelDiffTimes: number[] = [];
-  for (let i = 0; i < iterations; i++) {
-    const t0 = performance.now();
-    await computePixelDiff(baseImg, baseImg);
-    pixelDiffTimes.push(performance.now() - t0);
-  }
-  const pixelStats = calculatePercentiles(pixelDiffTimes);
-
-  // 2. Fast-path comparison latency (identical)
-  const fastPathTimes: number[] = [];
-  for (let i = 0; i < iterations; i++) {
-    const t0 = performance.now();
-    await compareScreenshots(baseImg, baseImg);
-    fastPathTimes.push(performance.now() - t0);
-  }
-  const fastPathStats = calculatePercentiles(fastPathTimes);
-
-  // 3. Minor-band comparison latency (sub-threshold noise)
-  const tier2Times: number[] = [];
-  for (let i = 0; i < iterations; i++) {
-    const t0 = performance.now();
-    await compareScreenshots(baseImg, minorImg);
-    tier2Times.push(performance.now() - t0);
-  }
-  const tier2Stats = calculatePercentiles(tier2Times);
-
-  const metrics: LatencyMetric[] = [
-    {
-      name: 'Pixel Diff Latency (1280x720)',
-      targetMs: 50,
-      actualMeanMs: pixelStats.mean,
-      p50Ms: pixelStats.p50,
-      p95Ms: pixelStats.p95,
-      passed: pixelStats.p95 <= 75,
-    },
-    {
-      name: 'Fast-Path Hook Latency (Identical)',
-      targetMs: 50,
-      actualMeanMs: fastPathStats.mean,
-      p50Ms: fastPathStats.p50,
-      p95Ms: fastPathStats.p95,
-      passed: fastPathStats.p95 <= 75,
-    },
-    {
-      name: 'Minor-Band Hook Latency (Sub-threshold Noise)',
-      targetMs: 250,
-      actualMeanMs: tier2Stats.mean,
-      p50Ms: tier2Stats.p50,
-      p95Ms: tier2Stats.p95,
-      passed: tier2Stats.p95 <= 300,
-    },
-  ];
-
-  return {
-    metrics,
-    allPassed: metrics.every(m => m.passed),
-  };
-}
-
-/**
- * Runs the accuracy benchmark suite checking True Positive Rate and False Positive Rate.
- */
-export async function runAccuracyBenchmark(sampleSize = 20): Promise<{
-  metrics: AccuracyMetric[];
-  truePositiveRate: number;
-  falsePositiveRate: number;
-  allPassed: boolean;
-}> {
-  const baseImg = createSyntheticUiPng();
-
-  let truePositives = 0;
-  let falseNegatives = 0;
-  let trueNegatives = 0;
-  let falsePositives = 0;
-
-  // Test True Positive: Actual Regressions (missing button, layout shift)
-  for (let i = 0; i < sampleSize; i++) {
-    const isShift = i % 2 === 0;
-    const brokenImg = createSyntheticUiPng({
-      removeButton: !isShift,
-      shiftLayout: isShift,
-    });
-
-    const result = await compareScreenshots(baseImg, brokenImg);
-    if (result.result === DiffResult.CHANGED || result.result === DiffResult.BROKEN) {
-      truePositives++;
-    } else {
-      falseNegatives++;
-    }
-  }
-
-  // Test False Positive: Benign changes (identical or sub-pixel noise)
-  for (let i = 0; i < sampleSize; i++) {
-    const isNoise = i % 2 === 0;
-    const benignImg = isNoise
-      ? createSyntheticUiPng({ addNoise: true, noiseAmount: 0.0003 })
-      : createSyntheticUiPng();
-
-    const result = await compareScreenshots(baseImg, benignImg);
-    if (result.result === DiffResult.IDENTICAL || result.result === DiffResult.MINOR) {
-      trueNegatives++;
-    } else {
-      falsePositives++;
-    }
-  }
-
-  const tpr = (truePositives / (truePositives + falseNegatives)) * 100;
-  const fpr = (falsePositives / (falsePositives + trueNegatives)) * 100;
-
-  const metrics: AccuracyMetric[] = [
-    {
-      name: 'True Positive Rate (Catches Real Regressions)',
-      targetPercent: 95.0,
-      actualPercent: tpr,
-      description: 'Percentage of genuine visual UI defects successfully caught',
-      passed: tpr >= 95.0,
-    },
-    {
-      name: 'False Positive Rate (Filters Benign Shifts)',
-      targetPercent: 5.0,
-      actualPercent: fpr,
-      description: 'Percentage of benign subpixel/anti-aliasing changes wrongly flagged',
-      passed: fpr <= 5.0,
-    },
-    {
-      name: 'Identical Match Precision',
-      targetPercent: 100.0,
-      actualPercent: ((trueNegatives / sampleSize) * 100),
-      description: 'Accuracy when comparing completely unchanged views',
-      passed: ((trueNegatives / sampleSize) * 100) >= 95.0,
-    },
-  ];
-
-  return {
-    metrics,
-    truePositiveRate: tpr,
-    falsePositiveRate: fpr,
-    allPassed: metrics.every(m => m.passed),
-  };
-}
-
-/**
- * Runs the complete benchmark suite (performance and accuracy).
- */
-export async function runAllBenchmarks(): Promise<BenchmarkReport> {
-  const perf = await runPerformanceBenchmark(10);
-  const acc = await runAccuracyBenchmark(20);
-
-  const totalBenchmarks = perf.metrics.length + acc.metrics.length;
-  const passedBenchmarks =
-    perf.metrics.filter(m => m.passed).length +
-    acc.metrics.filter(m => m.passed).length;
-
-  return {
-    timestamp: new Date().toISOString(),
-    performance: perf,
-    accuracy: acc,
-    summary: {
-      totalBenchmarks,
-      passedBenchmarks,
-      launchReady: perf.allPassed && acc.allPassed,
-    },
-  };
-}

package/src/capturer.ts

@@ -1,105 +0,0 @@
-import { chromium, type Browser } from 'playwright';
-import { promises as fs } from 'fs';
-import path from 'path';
-import type { CaptureResult, FlowRecording } from './types.js';
-
-let browserInstance: Browser | null = null;
-
-export async function getBrowser(): Promise<Browser> {
-  if (!browserInstance) {
-    browserInstance = await chromium.launch({ headless: true });
-  }
-  return browserInstance;
-}
-
-export async function closeBrowser(): Promise<void> {
-  if (browserInstance) {
-    await browserInstance.close();
-    browserInstance = null;
-  }
-}
-
-export async function captureCurrentState(options: {
-  url: string;
-  flow: FlowRecording;
-  outputDir: string;
-  viewport?: { width: number; height: number };
-}): Promise<CaptureResult> {
-  const { url, flow, outputDir, viewport = { width: 1280, height: 720 } } = options;
-  const browser = await getBrowser();
-  
-  const videoDir = path.join(outputDir, 'video');
-  await fs.mkdir(videoDir, { recursive: true });
-  
-  const context = await browser.newContext({
-    viewport,
-    recordVideo: { dir: videoDir }
-  });
-  
-  const page = await context.newPage();
-  const result: CaptureResult = {
-    screenshots: [],
-    screenshotPaths: [],
-    ariaSnapshots: [],
-    completedSteps: 0,
-    totalSteps: flow.steps.length,
-    errors: []
-  };
-
-  await fs.mkdir(outputDir, { recursive: true });
-
-  try {
-    let navigated = false;
-    for (let i = 0; i < flow.steps.length; i++) {
-      const step = flow.steps[i];
-      try {
-        if (!navigated && page.url() === 'about:blank' && step.action !== 'navigate') {
-          await page.goto(url, { waitUntil: 'networkidle' });
-          navigated = true;
-        }
-
-        if (step.action === 'navigate') {
-          await page.goto(step.url || url, { waitUntil: 'networkidle' });
-          navigated = true;
-        } else if (step.action === 'click' && step.selector) {
-          await page.click(step.selector, { timeout: step.timeout || 5000 });
-        } else if (step.action === 'type' && step.selector && step.value !== undefined) {
-          await page.fill(step.selector, step.value, { timeout: step.timeout || 5000 });
-        } else if (step.action === 'scroll' && step.position) {
-          await page.mouse.wheel(step.position.x, step.position.y);
-        } else if (step.action === 'wait') {
-          await page.waitForTimeout(step.timeout || 1000);
-        }
-
-        await page.waitForTimeout(500);
-        
-        const screenshotPath = path.join(outputDir, `step_${i}.png`);
-        const buffer = await page.screenshot({ path: screenshotPath });
-
-        // Semantic DOM state for the classifier tier (diffable as text)
-        let aria = '';
-        try {
-          aria = await page.locator('body').ariaSnapshot();
-        } catch {
-          aria = '';
-        }
-        await fs.writeFile(path.join(outputDir, `aria_step_${i}.yml`), aria, 'utf-8');
-
-        result.screenshots.push(buffer);
-        result.screenshotPaths.push(screenshotPath);
-        result.ariaSnapshots.push(aria);
-        result.completedSteps++;
-      } catch (err: any) {
-        result.errors.push(`Step ${i} (${step.action}) failed: ${err.message}`);
-        break;
-      }
-    }
-  } finally {
-    const videoObj = await page.video();
-    const videoPath = videoObj ? await videoObj.path() : undefined;
-    result.videoPath = videoPath;
-    await context.close();
-  }
-
-  return result;
-}

package/src/classifier.ts

@@ -1,258 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import os from 'node:os';
-import { diffAriaSnapshots } from './domdiff.js';
-import type { ClassifierConfig } from './types.js';
-
-export interface ClassifierVerdict {
-  intentional: boolean;
-  /** 0..1 certainty reported by the classifier */
-  confidence: number;
-  /** One sentence: what changed and — if not intentional — how to fix it */
-  reasoning: string;
-}
-
-/** Metadata about the flow being classified (the hosted API uses it for the flow gate + usage log) */
-export interface ClassifyMeta {
-  flowName?: string;
-  flowHash?: string;
-  step?: number;
-}
-
-/**
- * Thrown when the hosted API returns 429 — either the free-plan flow limit
- * (distinct flows) or the monthly abuse cap. Carries the payload surfaced
- * in agent feedback and UI ("10 of 10 flows guarded — upgrade for unlimited").
- */
-export class FlowLimitError extends Error {
-  readonly kind: 'flow_limit_reached' | 'monthly_cap_reached';
-  readonly flowsUsed?: number;
-  readonly flowLimit?: number;
-  readonly upgradeUrl?: string;
-
-  constructor(body: { error?: string; flowsUsed?: number; flowLimit?: number; upgradeUrl?: string }) {
-    super(body.error ?? 'limit_reached');
-    this.name = 'FlowLimitError';
-    this.kind = body.error === 'monthly_cap_reached' ? 'monthly_cap_reached' : 'flow_limit_reached';
-    this.flowsUsed = body.flowsUsed;
-    this.flowLimit = body.flowLimit;
-    this.upgradeUrl = body.upgradeUrl;
-  }
-}
-
-const SYSTEM_PROMPT = [
-  'You are a strict UI regression classifier for an automated QA tool that guards AI coding agents.',
-  'You will be given: (1) a structural diff of the page accessibility tree between the accepted baseline and the current state after a code change, and (2) optional context about what the developer/agent was asked to do in the latest turn.',
-  'The baseline is the ACCEPTED, WORKING state. Removed or broken elements (buttons, forms, prices, text, navigation) are regressions even if the remaining page looks coherent.',
-  'If no context is provided, be conservative: significant structural changes should be classified as not intentional.',
-  'If context IS provided and the diff implements exactly what the context asked for, classify it as intentional with high confidence — a requested change is not a regression.',
-  'Benign changes (reordered identical items, minor text edits, added requested features) are intentional.',
-  'Answer with ONLY a JSON object, no other text:',
-  '{"intentional": <true|false>, "confidence": <number between 0 and 1>, "reasoning": "<exactly one sentence: what changed, and if not intentional, the specific fix>"}',
-].join('\n');
-
-/**
- * Classifies a visual change as intentional or a regression. Provider
- * 'kintsugi' POSTs to the hosted classifier API (which owns the model and
- * the flow gate); 'openai-compatible' sends the chat-completions prompt
- * below to any OpenAI-compatible endpoint.
- */
-export async function classifyChange(options: {
-  baselineAria: string;
-  currentAria: string;
-  context?: string;
-  config: ClassifierConfig;
-  apiToken: string;
-  meta?: ClassifyMeta;
-  logger?: { info: (msg: string, data?: unknown) => void; warn: (msg: string, data?: unknown) => void };
-}): Promise<ClassifierVerdict> {
-  if (options.config.provider === 'kintsugi') {
-    return classifyViaKintsugi(options);
-  }
-  return classifyViaOpenAiCompatible(options);
-}
-
-/** Hosted kintsugi API path — the server computes the diff, picks the model, and enforces the flow gate. */
-async function classifyViaKintsugi(options: {
-  baselineAria: string;
-  currentAria: string;
-  context?: string;
-  config: ClassifierConfig;
-  apiToken: string;
-  meta?: ClassifyMeta;
-}): Promise<ClassifierVerdict> {
-  const { baselineAria, currentAria, context, config, apiToken, meta } = options;
-  const url = `${config.endpoint.replace(/\/$/, '')}/classify`;
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), config.timeoutMs);
-  let response: Response;
-  try {
-    response = await fetch(url, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        Authorization: `Bearer ${apiToken}`,
-      },
-      body: JSON.stringify({ baselineAria, currentAria, context: context ?? null, meta: meta ?? null }),
-      signal: controller.signal,
-    });
-  } finally {
-    clearTimeout(timer);
-  }
-
-  if (response.status === 429) {
-    const body = await response.json().catch(() => ({}));
-    throw new FlowLimitError(body);
-  }
-  if (!response.ok) {
-    const detail = await response.text().catch(() => '');
-    throw new Error(`kintsugi api ${response.status}: ${detail.slice(0, 300)}`);
-  }
-  const payload = (await response.json()) as Partial<ClassifierVerdict>;
-  if (typeof payload.intentional !== 'boolean' || typeof payload.confidence !== 'number') {
-    throw new Error(`unexpected kintsugi api response: ${JSON.stringify(payload).slice(0, 200)}`);
-  }
-  return {
-    intentional: payload.intentional,
-    confidence: payload.confidence,
-    reasoning: payload.reasoning ?? '',
-  };
-}
-
-async function classifyViaOpenAiCompatible(options: {
-  baselineAria: string;
-  currentAria: string;
-  context?: string;
-  config: ClassifierConfig;
-  apiToken: string;
-  logger?: { info: (msg: string, data?: unknown) => void; warn: (msg: string, data?: unknown) => void };
-}): Promise<ClassifierVerdict> {
-  const { baselineAria, currentAria, context, config, apiToken, logger } = options;
-
-  const diff = diffAriaSnapshots(baselineAria, currentAria);
-  const userText = [
-    'Accessibility-tree diff (baseline → current). Lines prefixed "REMOVED:" were removed from the page, "ADDED:" were added:',
-    diff,
-    '',
-    context
-      ? `Context from the latest turn (what the agent was asked to do):\n"${context}"`
-      : 'No context available about the intended change.',
-    '',
-    'Classify this change as intentional or not. Reply with ONLY the JSON object.',
-  ].join('\n');
-
-  const body = {
-    model: config.model,
-    temperature: 0,
-    max_tokens: 120,
-    messages: [
-      { role: 'system', content: SYSTEM_PROMPT },
-      { role: 'user', content: userText },
-    ],
-  };
-
-  const url = `${config.endpoint.replace(/\/$/, '')}/chat/completions`;
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), config.timeoutMs);
-  let response: Response;
-  try {
-    response = await fetch(url, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        Authorization: `Bearer ${apiToken}`,
-      },
-      body: JSON.stringify(body),
-      signal: controller.signal,
-    });
-  } finally {
-    clearTimeout(timer);
-  }
-
-  if (!response.ok) {
-    const detail = await response.text().catch(() => '');
-    throw new Error(`classifier endpoint ${response.status}: ${detail.slice(0, 300)}`);
-  }
-
-  const payload = JSON.parse(await response.text()) as {
-    choices?: { message?: { content?: string } }[];
-  };
-  const content = payload.choices?.[0]?.message?.content ?? '';
-  logger?.info('classifier responded', { content: content.slice(0, 300) });
-  return parseVerdict(content);
-}
-
-/** Robustly extracts {intentional, confidence, reasoning} from the model output. Fails safe to not-intentional. */
-export function parseVerdict(content: string): ClassifierVerdict {
-  const text = content.trim();
-
-  // Preferred: the JSON object we asked for
-  const jsonMatch = text.match(/\{[\s\S]*?\}/);
-  if (jsonMatch) {
-    try {
-      const parsed = JSON.parse(jsonMatch[0]) as {
-        intentional?: boolean;
-        confidence?: number;
-        reasoning?: string;
-      };
-      if (typeof parsed.intentional === 'boolean') {
-        return {
-          intentional: parsed.intentional,
-          confidence: typeof parsed.confidence === 'number'
-            ? Math.min(1, Math.max(0, parsed.confidence))
-            : 0.5,
-          reasoning: typeof parsed.reasoning === 'string' && parsed.reasoning.trim()
-            ? parsed.reasoning.trim()
-            : 'No reasoning provided by classifier.',
-        };
-      }
-    } catch {
-      // fall through to fail-safe
-    }
-  }
-
-  // Unparseable — fail safe: treat as an unresolved regression
-  return {
-    intentional: false,
-    confidence: 0.5,
-    reasoning: 'Classifier output was unparseable; treat this change as a regression and inspect the aria diff manually.',
-  };
-}
-
-/**
- * Resolves the API token: process env first, then the project's
- * `<projectDir>/.kintsugi/.env`, then the machine-global `~/.kintsugi/.env`
- * (so one key works across all projects).
- */
-export function resolveApiToken(tokenEnvVar: string, projectDir?: string): string | undefined {
-  if (process.env[tokenEnvVar]) return process.env[tokenEnvVar];
-  if (projectDir) loadKintsugiEnv(projectDir);
-  loadEnvFile(globalKintsugiEnvPath());
-  return process.env[tokenEnvVar];
-}
-
-/** Path of the machine-global kintsugi env file — where the hosted API key lives. */
-export function globalKintsugiEnvPath(): string {
-  return path.join(os.homedir(), '.kintsugi', '.env');
-}
-
-/** Loads KEY=VALUE pairs from <projectDir>/.kintsugi/.env into process.env (never overrides). */
-export function loadKintsugiEnv(projectDir: string): void {
-  loadEnvFile(path.join(projectDir, '.kintsugi', '.env'));
-}
-
-function loadEnvFile(envPath: string): void {
-  let content: string;
-  try {
-    content = fs.readFileSync(envPath, 'utf-8');
-  } catch {
-    return;
-  }
-  for (const line of content.split('\n')) {
-    const match = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/);
-    if (!match) continue;
-    const key = match[1];
-    const value = match[2].replace(/^["']|["']$/g, '');
-    if (!(key in process.env)) process.env[key] = value;
-  }
-}