chati-dev 3.2.4 → 3.3.0

package/src/quality/metrics-collector.js

@@ -0,0 +1,283 @@
+/**
+ * @fileoverview Quality metrics collector and trend analysis.
+ *
+ * Persists quality metrics over time and calculates trends
+ * (improving, stable, declining) via linear regression.
+ *
+ * Constitution Article XIV — Framework Registry Governance.
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { join, dirname } from 'path';
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+/** Metrics storage filename. */
+const METRICS_FILE = 'quality-history.json';
+
+/** Metrics directory relative to project root. */
+const METRICS_DIR = '.chati/metrics';
+
+/** Maximum metrics entries to keep per project. */
+export const MAX_METRICS_ENTRIES = 500;
+
+/** Valid metric types. */
+export const METRIC_TYPES = {
+  QA_SCORE: 'qa_score',
+  HEALTH_SCORE: 'health_score',
+  TEST_COUNT: 'test_count',
+  COVERAGE: 'coverage',
+  LINT_ERRORS: 'lint_errors',
+  BUILD_TIME: 'build_time',
+};
+
+/** Trend direction values. */
+export const TREND_DIRECTION = {
+  IMPROVING: 'improving',
+  STABLE: 'stable',
+  DECLINING: 'declining',
+};
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Get the metrics file path for a project.
+ *
+ * @param {string} projectDir
+ * @returns {string}
+ */
+function getMetricsPath(projectDir) {
+  return join(projectDir, METRICS_DIR, METRICS_FILE);
+}
+
+/**
+ * Load metrics from disk.
+ *
+ * @param {string} projectDir
+ * @returns {{ metrics: object[] }}
+ */
+function loadMetrics(projectDir) {
+  const filePath = getMetricsPath(projectDir);
+  if (!existsSync(filePath)) {
+    return { metrics: [] };
+  }
+
+  try {
+    const raw = readFileSync(filePath, 'utf-8');
+    const data = JSON.parse(raw);
+    return { metrics: Array.isArray(data.metrics) ? data.metrics : [] };
+  } catch {
+    return { metrics: [] };
+  }
+}
+
+/**
+ * Save metrics to disk.
+ *
+ * @param {string} projectDir
+ * @param {{ metrics: object[] }} data
+ */
+function saveMetrics(projectDir, data) {
+  const filePath = getMetricsPath(projectDir);
+  const dir = dirname(filePath);
+
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+
+  // Trim to MAX_METRICS_ENTRIES
+  if (data.metrics.length > MAX_METRICS_ENTRIES) {
+    data.metrics = data.metrics.slice(-MAX_METRICS_ENTRIES);
+  }
+
+  writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * @typedef {object} MetricEntry
+ * @property {string} type - Metric type (from METRIC_TYPES)
+ * @property {number} value - Metric value
+ * @property {string} [agent] - Agent that produced the metric
+ * @property {string} [sessionId] - Session identifier
+ * @property {string} timestamp - ISO timestamp
+ */
+
+/**
+ * Record a quality metric.
+ *
+ * @param {string} projectDir
+ * @param {{ type: string, value: number, agent?: string, sessionId?: string }} metric
+ * @returns {{ recorded: boolean, total: number }}
+ */
+export function recordMetric(projectDir, metric) {
+  if (!metric || typeof metric.value !== 'number') {
+    return { recorded: false, total: 0 };
+  }
+
+  const validTypes = Object.values(METRIC_TYPES);
+  if (!validTypes.includes(metric.type)) {
+    return { recorded: false, total: 0 };
+  }
+
+  const data = loadMetrics(projectDir);
+
+  data.metrics.push({
+    type: metric.type,
+    value: metric.value,
+    agent: metric.agent || null,
+    sessionId: metric.sessionId || null,
+    timestamp: new Date().toISOString(),
+  });
+
+  saveMetrics(projectDir, data);
+
+  return { recorded: true, total: data.metrics.length };
+}
+
+/**
+ * Get metrics history with optional filtering.
+ *
+ * @param {string} projectDir
+ * @param {{ type?: string, agent?: string, limit?: number, since?: string }} [options={}]
+ * @returns {{ metrics: MetricEntry[], trend: { direction: string, slope: number, confidence: number } }}
+ */
+export function getMetricsHistory(projectDir, options = {}) {
+  const data = loadMetrics(projectDir);
+  let metrics = data.metrics;
+
+  // Filter by type
+  if (options.type) {
+    metrics = metrics.filter(m => m.type === options.type);
+  }
+
+  // Filter by agent
+  if (options.agent) {
+    metrics = metrics.filter(m => m.agent === options.agent);
+  }
+
+  // Filter by time
+  if (options.since) {
+    const sinceDate = new Date(options.since).getTime();
+    metrics = metrics.filter(m => new Date(m.timestamp).getTime() >= sinceDate);
+  }
+
+  // Apply limit
+  if (options.limit && options.limit > 0) {
+    metrics = metrics.slice(-options.limit);
+  }
+
+  // Calculate trend
+  const trend = calculateTrend(metrics);
+
+  return { metrics, trend };
+}
+
+/**
+ * Calculate trend from a series of metrics using linear regression.
+ *
+ * @param {MetricEntry[]} metrics
+ * @returns {{ direction: string, slope: number, confidence: number }}
+ */
+export function calculateTrend(metrics) {
+  if (!metrics || metrics.length < 3) {
+    return { direction: TREND_DIRECTION.STABLE, slope: 0, confidence: 0 };
+  }
+
+  const values = metrics.map(m => m.value);
+  const n = values.length;
+
+  // Simple linear regression: y = mx + b
+  let sumX = 0;
+  let sumY = 0;
+  let sumXY = 0;
+  let sumX2 = 0;
+  let sumY2 = 0;
+
+  for (let i = 0; i < n; i++) {
+    sumX += i;
+    sumY += values[i];
+    sumXY += i * values[i];
+    sumX2 += i * i;
+    sumY2 += values[i] * values[i];
+  }
+
+  const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
+
+  // Calculate R² (coefficient of determination)
+  const meanY = sumY / n;
+  let ssRes = 0;
+  let ssTot = 0;
+
+  for (let i = 0; i < n; i++) {
+    const predicted = meanY + slope * (i - (n - 1) / 2);
+    ssRes += (values[i] - predicted) ** 2;
+    ssTot += (values[i] - meanY) ** 2;
+  }
+
+  const r2 = ssTot > 0 ? 1 - ssRes / ssTot : 0;
+  const confidence = Math.max(0, Math.min(1, Math.abs(r2)));
+
+  // Determine direction
+  // Normalize slope by mean to get relative change
+  const normalizedSlope = meanY !== 0 ? slope / Math.abs(meanY) : slope;
+  const threshold = 0.02; // 2% change per step is significant
+
+  let direction;
+  if (normalizedSlope > threshold && confidence > 0.3) {
+    direction = TREND_DIRECTION.IMPROVING;
+  } else if (normalizedSlope < -threshold && confidence > 0.3) {
+    direction = TREND_DIRECTION.DECLINING;
+  } else {
+    direction = TREND_DIRECTION.STABLE;
+  }
+
+  return {
+    direction,
+    slope: Math.round(slope * 1000) / 1000,
+    confidence: Math.round(confidence * 100) / 100,
+  };
+}
+
+/**
+ * Get a quality dashboard summary.
+ *
+ * @param {string} projectDir
+ * @returns {{ latestScores: Record<string, number>, trends: Record<string, object>, totalMetrics: number, lastUpdated: string|null }}
+ */
+export function getQualityDashboard(projectDir) {
+  const data = loadMetrics(projectDir);
+  const metrics = data.metrics;
+
+  if (metrics.length === 0) {
+    return { latestScores: {}, trends: {}, totalMetrics: 0, lastUpdated: null };
+  }
+
+  // Get latest value for each metric type
+  const latestScores = {};
+  const trends = {};
+
+  for (const type of Object.values(METRIC_TYPES)) {
+    const typeMetrics = metrics.filter(m => m.type === type);
+    if (typeMetrics.length > 0) {
+      latestScores[type] = typeMetrics[typeMetrics.length - 1].value;
+      trends[type] = calculateTrend(typeMetrics);
+    }
+  }
+
+  const lastUpdated = metrics[metrics.length - 1]?.timestamp || null;
+
+  return {
+    latestScores,
+    trends,
+    totalMetrics: metrics.length,
+    lastUpdated,
+  };
+}

package/src/quality/test-runner.js

@@ -0,0 +1,368 @@
+/**
+ * @fileoverview Real code execution validation.
+ *
+ * Detects test runners, executes tests and lint commands,
+ * and parses output for structured results.
+ *
+ * Constitution Article XIV — Framework Registry Governance.
+ */
+
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { execSync } from 'child_process';
+
+// ---------------------------------------------------------------------------
+// Test Runner Detection
+// ---------------------------------------------------------------------------
+
+/**
+ * @typedef {object} TestDetection
+ * @property {string|null} command - Test command to run
+ * @property {string} runner - Detected runner name (node:test|jest|vitest|mocha|unknown)
+ * @property {boolean} detected - Whether a test setup was found
+ */
+
+/**
+ * Detect the test command and runner for a project.
+ *
+ * Detection order:
+ * 1. package.json scripts.test
+ * 2. vitest.config.* presence
+ * 3. jest.config.* presence
+ * 4. .mocharc.* presence
+ *
+ * @param {string} projectDir - Project root directory
+ * @returns {TestDetection}
+ */
+export function detectTestCommand(projectDir) {
+  const pkgPath = join(projectDir, 'package.json');
+
+  // Check package.json scripts.test
+  if (existsSync(pkgPath)) {
+    try {
+      const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
+      const testScript = pkg.scripts?.test;
+
+      if (testScript && testScript !== 'echo "Error: no test specified" && exit 1') {
+        const runner = detectRunnerFromScript(testScript);
+        return { command: 'npm test', runner, detected: true };
+      }
+    } catch {
+      // Malformed package.json — continue detection
+    }
+  }
+
+  // Check for vitest.config.*
+  const vitestConfigs = ['vitest.config.ts', 'vitest.config.js', 'vitest.config.mts'];
+  for (const cfg of vitestConfigs) {
+    if (existsSync(join(projectDir, cfg))) {
+      return { command: 'npx vitest run', runner: 'vitest', detected: true };
+    }
+  }
+
+  // Check for jest.config.*
+  const jestConfigs = ['jest.config.ts', 'jest.config.js', 'jest.config.json'];
+  for (const cfg of jestConfigs) {
+    if (existsSync(join(projectDir, cfg))) {
+      return { command: 'npx jest', runner: 'jest', detected: true };
+    }
+  }
+
+  // Check for .mocharc.*
+  const mochaConfigs = ['.mocharc.yml', '.mocharc.yaml', '.mocharc.json', '.mocharc.js'];
+  for (const cfg of mochaConfigs) {
+    if (existsSync(join(projectDir, cfg))) {
+      return { command: 'npx mocha', runner: 'mocha', detected: true };
+    }
+  }
+
+  return { command: null, runner: 'unknown', detected: false };
+}
+
+/**
+ * Detect runner from a test script string.
+ *
+ * @param {string} script
+ * @returns {string}
+ */
+function detectRunnerFromScript(script) {
+  if (/node\s+--test/.test(script)) return 'node:test';
+  if (/vitest/.test(script)) return 'vitest';
+  if (/jest/.test(script)) return 'jest';
+  if (/mocha/.test(script)) return 'mocha';
+  if (/tap/.test(script)) return 'tap';
+  return 'unknown';
+}
+
+// ---------------------------------------------------------------------------
+// Test Execution
+// ---------------------------------------------------------------------------
+
+/**
+ * @typedef {object} TestResult
+ * @property {boolean} success - All tests passed
+ * @property {number} total - Total test count
+ * @property {number} passed - Tests passed
+ * @property {number} failed - Tests failed
+ * @property {number} skipped - Tests skipped
+ * @property {number} duration - Execution time in ms
+ * @property {string} rawOutput - Full command output
+ * @property {string} runner - Detected runner
+ */
+
+/**
+ * Run tests in a project directory.
+ *
+ * @param {string} projectDir
+ * @param {{ timeout?: number, command?: string }} [options={}]
+ * @returns {TestResult}
+ */
+export function runTests(projectDir, options = {}) {
+  const detection = detectTestCommand(projectDir);
+  const command = options.command || detection.command;
+  const timeout = options.timeout || 120_000;
+
+  if (!command) {
+    return {
+      success: false,
+      total: 0,
+      passed: 0,
+      failed: 0,
+      skipped: 0,
+      duration: 0,
+      rawOutput: 'No test command detected',
+      runner: 'unknown',
+    };
+  }
+
+  const start = Date.now();
+  let rawOutput = '';
+  let exitCode = 0;
+
+  try {
+    rawOutput = execSync(command, {
+      cwd: projectDir,
+      encoding: 'utf-8',
+      timeout,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+  } catch (err) {
+    exitCode = err.status || 1;
+    rawOutput = (err.stdout || '') + (err.stderr || '');
+  }
+
+  const duration = Date.now() - start;
+  const parsed = parseTestOutput(rawOutput, detection.runner);
+
+  return {
+    success: exitCode === 0 && parsed.failed === 0,
+    total: parsed.total,
+    passed: parsed.passed,
+    failed: parsed.failed,
+    skipped: parsed.skipped,
+    duration,
+    rawOutput: rawOutput.slice(0, 5000),
+    runner: detection.runner,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Lint Execution
+// ---------------------------------------------------------------------------
+
+/**
+ * @typedef {object} LintResult
+ * @property {boolean} success - No lint errors
+ * @property {number} errors - Error count
+ * @property {number} warnings - Warning count
+ * @property {string} rawOutput - Full command output
+ */
+
+/**
+ * Run lint in a project directory.
+ *
+ * @param {string} projectDir
+ * @param {{ timeout?: number, command?: string }} [options={}]
+ * @returns {LintResult}
+ */
+export function runLint(projectDir, options = {}) {
+  const timeout = options.timeout || 60_000;
+  let command = options.command;
+
+  // Auto-detect lint command
+  if (!command) {
+    const pkgPath = join(projectDir, 'package.json');
+    if (existsSync(pkgPath)) {
+      try {
+        const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
+        if (pkg.scripts?.lint) {
+          command = 'npm run lint';
+        }
+      } catch {
+        // ignore
+      }
+    }
+  }
+
+  if (!command) {
+    return { success: true, errors: 0, warnings: 0, rawOutput: 'No lint command detected' };
+  }
+
+  let rawOutput = '';
+  let exitCode = 0;
+
+  try {
+    rawOutput = execSync(command, {
+      cwd: projectDir,
+      encoding: 'utf-8',
+      timeout,
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+  } catch (err) {
+    exitCode = err.status || 1;
+    rawOutput = (err.stdout || '') + (err.stderr || '');
+  }
+
+  const { errors, warnings } = parseLintOutput(rawOutput);
+
+  return {
+    success: exitCode === 0 && errors === 0,
+    errors,
+    warnings,
+    rawOutput: rawOutput.slice(0, 5000),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Output Parsing
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse test output to extract counts.
+ *
+ * @param {string} output
+ * @param {string} runner
+ * @returns {{ total: number, passed: number, failed: number, skipped: number }}
+ */
+export function parseTestOutput(output, runner) {
+  if (!output) {
+    return { total: 0, passed: 0, failed: 0, skipped: 0 };
+  }
+
+  switch (runner) {
+    case 'node:test':
+      return parseNodeTestOutput(output);
+    case 'jest':
+      return parseJestOutput(output);
+    case 'vitest':
+      return parseVitestOutput(output);
+    case 'mocha':
+      return parseMochaOutput(output);
+    default:
+      return parseGenericOutput(output);
+  }
+}
+
+/**
+ * Parse node:test TAP-like output.
+ */
+function parseNodeTestOutput(output) {
+  const testsMatch = output.match(/# tests (\d+)/);
+  const passMatch = output.match(/# pass (\d+)/);
+  const failMatch = output.match(/# fail (\d+)/);
+  const skipMatch = output.match(/# skipped (\d+)/);
+
+  const total = testsMatch ? parseInt(testsMatch[1], 10) : 0;
+  const passed = passMatch ? parseInt(passMatch[1], 10) : 0;
+  const failed = failMatch ? parseInt(failMatch[1], 10) : 0;
+  const skipped = skipMatch ? parseInt(skipMatch[1], 10) : 0;
+
+  return { total, passed, failed, skipped };
+}
+
+/**
+ * Parse Jest output.
+ */
+function parseJestOutput(output) {
+  // Jest format: "Tests:  3 failed, 47 passed, 50 total"
+  const totalMatch = output.match(/(\d+)\s+total/);
+  const passedMatch = output.match(/(\d+)\s+passed/);
+  const failMatch = output.match(/(\d+)\s+failed/);
+  const skipMatch = output.match(/(\d+)\s+skipped/);
+
+  const total = totalMatch ? parseInt(totalMatch[1], 10) : 0;
+  const passed = passedMatch ? parseInt(passedMatch[1], 10) : 0;
+  const failed = failMatch ? parseInt(failMatch[1], 10) : 0;
+  const skipped = skipMatch ? parseInt(skipMatch[1], 10) : 0;
+
+  return { total, passed, failed, skipped };
+}
+
+/**
+ * Parse Vitest output.
+ */
+function parseVitestOutput(output) {
+  const passMatch = output.match(/(\d+)\s+passed/);
+  const failMatch = output.match(/(\d+)\s+failed/);
+  const skipMatch = output.match(/(\d+)\s+skipped/);
+
+  const passed = passMatch ? parseInt(passMatch[1], 10) : 0;
+  const failed = failMatch ? parseInt(failMatch[1], 10) : 0;
+  const skipped = skipMatch ? parseInt(skipMatch[1], 10) : 0;
+  const total = passed + failed + skipped;
+
+  return { total, passed, failed, skipped };
+}
+
+/**
+ * Parse Mocha output.
+ */
+function parseMochaOutput(output) {
+  const passMatch = output.match(/(\d+)\s+passing/);
+  const failMatch = output.match(/(\d+)\s+failing/);
+  const pendMatch = output.match(/(\d+)\s+pending/);
+
+  const passed = passMatch ? parseInt(passMatch[1], 10) : 0;
+  const failed = failMatch ? parseInt(failMatch[1], 10) : 0;
+  const skipped = pendMatch ? parseInt(pendMatch[1], 10) : 0;
+  const total = passed + failed + skipped;
+
+  return { total, passed, failed, skipped };
+}
+
+/**
+ * Generic fallback parser.
+ */
+function parseGenericOutput(output) {
+  const passMatch = output.match(/(\d+)\s+pass/i);
+  const failMatch = output.match(/(\d+)\s+fail/i);
+
+  const passed = passMatch ? parseInt(passMatch[1], 10) : 0;
+  const failed = failMatch ? parseInt(failMatch[1], 10) : 0;
+  const total = passed + failed;
+
+  return { total, passed, failed, skipped: 0 };
+}
+
+/**
+ * Parse lint output for error/warning counts.
+ *
+ * @param {string} output
+ * @returns {{ errors: number, warnings: number }}
+ */
+function parseLintOutput(output) {
+  // ESLint format: "X problems (Y errors, Z warnings)"
+  const eslintMatch = output.match(/(\d+)\s+problems?\s*\((\d+)\s+errors?,\s*(\d+)\s+warnings?\)/);
+  if (eslintMatch) {
+    return {
+      errors: parseInt(eslintMatch[2], 10),
+      warnings: parseInt(eslintMatch[3], 10),
+    };
+  }
+
+  // Count error/warning lines
+  const errorLines = (output.match(/error/gi) || []).length;
+  const warningLines = (output.match(/warning/gi) || []).length;
+
+  return { errors: errorLines, warnings: warningLines };
+}