cache-verdict 0.0.0-stage → 0.1.0

package/lib/report.mjs

@@ -0,0 +1,144 @@
+function median(values) {
+  if (values.length === 0) return null;
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = Math.floor(sorted.length / 2);
+  if (sorted.length % 2 === 1) return sorted[middle];
+  return (sorted[middle - 1] + sorted[middle]) / 2;
+}
+
+function finite(value) {
+  return typeof value === 'number' && Number.isFinite(value) ? value : null;
+}
+
+function sampleTiming(sample) {
+  const buildMs = finite(sample.buildMs);
+  const restoreMs = finite(sample.restoreMs);
+  const totalMs = finite(sample.totalMs) ?? (buildMs !== null && restoreMs !== null ? buildMs + restoreMs : null);
+  return {
+    buildMs,
+    restoreMs,
+    totalMs,
+    hit: sample.hit === true || sample.temperature === 'warm',
+    key: typeof sample.key === 'string' ? sample.key : 'unknown',
+  };
+}
+
+function round(value) {
+  return value === null ? null : Math.round(value * 1000) / 1000;
+}
+
+/**
+ * Turn ledger samples into the numbers and single verdict the report prints.
+ *
+ * @param {Array<object>} samples samples read from the JSONL ledger
+ * @param {{ hasCachePaths?: boolean, storedKeys?: number, storedSizeBytes?: number }} [context]
+ */
+export function analyzeSamples(samples, context = {}) {
+  const hasCachePaths = context.hasCachePaths !== false;
+  const observations = samples.map(sampleTiming).filter((sample) => sample.totalMs !== null);
+
+  const cold = observations.filter((sample) => !sample.hit);
+  const warm = observations.filter((sample) => sample.hit);
+  const keys = new Set(observations.map((sample) => sample.key));
+
+  const coldTotals = cold.map((sample) => sample.totalMs);
+  const warmTotals = warm.map((sample) => sample.totalMs);
+  const restoreCosts = warm
+    .map((sample) => sample.restoreMs)
+    .filter((value) => value !== null);
+
+  const medianColdMs = median(coldTotals);
+  const medianWarmMs = median(warmTotals);
+  const restoreMs = median(restoreCosts);
+  const netMsPerHit =
+    medianColdMs !== null && medianWarmMs !== null ? medianColdMs - medianWarmMs : null;
+
+  let verdict;
+  let moreRunsNeeded = false;
+  let reason;
+
+  if (!hasCachePaths) {
+    verdict = 'cache-unobserved';
+    reason = 'cachePath is empty, so there is nothing to measure';
+  } else if (observations.length === 0) {
+    verdict = 'cache-unobserved';
+    moreRunsNeeded = true;
+    reason = 'no samples have been recorded yet; run the build once';
+  } else if (warm.length === 0) {
+    if (observations.length < 2) {
+      verdict = 'cache-unobserved';
+      moreRunsNeeded = true;
+      reason = 'one cold sample is not enough; run again to collect a warm sample';
+    } else {
+      verdict = 'cache-never-hits';
+      reason = `all ${observations.length} samples missed the cache`;
+    }
+  } else if (cold.length === 0) {
+    verdict = 'cache-unobserved';
+    moreRunsNeeded = true;
+    reason = 'no cold baseline has been recorded; add one with "cache-verdict run --cold"';
+  } else if (keys.size > 1) {
+    verdict = 'cache-churns';
+    reason = `${keys.size} distinct keys across ${observations.length} samples; the cache key changes too often to accumulate hits`;
+  } else if (netMsPerHit !== null && netMsPerHit > 0) {
+    verdict = 'cache-pays';
+    reason = `warm builds are ${netMsPerHit.toFixed(1)} ms faster per hit after paying the restore cost`;
+  } else {
+    verdict = 'cache-costs';
+    reason = `a warm build is not faster than cold (${netMsPerHit?.toFixed(1) ?? 'unknown'} ms per hit)`;
+  }
+
+  return {
+    runs: observations.length,
+    hits: warm.length,
+    misses: cold.length,
+    hitRate: observations.length === 0 ? null : warm.length / observations.length,
+    coldRuns: cold.length,
+    warmRuns: warm.length,
+    uniqueKeys: keys.size,
+    medianColdMs: round(medianColdMs),
+    medianWarmMs: round(medianWarmMs),
+    restoreMs: round(restoreMs),
+    netMsPerHit: round(netMsPerHit),
+    storedKeys: context.storedKeys ?? 0,
+    cacheSizeBytes: context.storedSizeBytes ?? 0,
+    verdict,
+    moreRunsNeeded,
+    reason,
+  };
+}
+
+function formatMs(value) {
+  return value === null || value === undefined ? 'n/a' : `${value.toFixed(1)} ms`;
+}
+
+function formatBytes(value) {
+  if (!Number.isFinite(value) || value <= 0) return '0 B';
+  if (value < 1024) return `${value} B`;
+  const units = ['KiB', 'MiB', 'GiB', 'TiB'];
+  let scaled = value;
+  let unit = -1;
+  do {
+    scaled /= 1024;
+    unit += 1;
+  } while (scaled >= 1024 && unit < units.length - 1);
+  return `${scaled.toFixed(1)} ${units[unit]}`;
+}
+
+export function formatReport(analysis) {
+  const hitRate =
+    analysis.hitRate === null ? 'n/a' : `${(analysis.hitRate * 100).toFixed(1)}%`;
+  const storedKeys = analysis.storedKeys === 1 ? 'stored key' : 'stored keys';
+  const lines = [
+    `Runs: ${analysis.runs} (${analysis.hits} hit${analysis.hits === 1 ? '' : 's'}, ${analysis.misses} miss${analysis.misses === 1 ? '' : 'es'})`,
+    `Hit rate: ${hitRate}`,
+    `Median cold total: ${formatMs(analysis.medianColdMs)}`,
+    `Median warm total: ${formatMs(analysis.medianWarmMs)}`,
+    `Restore cost: ${formatMs(analysis.restoreMs)}`,
+    `Net ms per hit: ${formatMs(analysis.netMsPerHit)}`,
+    `Cache size: ${formatBytes(analysis.cacheSizeBytes)} across ${analysis.storedKeys} ${storedKeys}`,
+    `Verdict: ${analysis.verdict}`,
+  ];
+  if (analysis.reason) lines.push(`Reason: ${analysis.reason}`);
+  return `${lines.join('\n')}\n`;
+}

package/lib/run.mjs

@@ -0,0 +1,204 @@
+import { randomUUID } from 'node:crypto';
+import { spawnSync } from 'node:child_process';
+import path from 'node:path';
+
+import { loadConfig } from './config.mjs';
+import { resolvePathPatterns } from './glob.mjs';
+import { computeInputKey } from './key.mjs';
+import {
+  appendLedgerSample,
+  ensureWorkDirectory,
+  movePath,
+  pathExists,
+  pathSize,
+  readStoredCache,
+  removePath,
+  restoreStoredCache,
+  storeCache,
+  storeKeyDirectory,
+  tempDirectory,
+  touchStoredCache,
+  workDirectory,
+} from './ledger.mjs';
+
+function nowMs() {
+  return Number(process.hrtime.bigint()) / 1e6;
+}
+
+function roundMs(value) {
+  return Math.round(value * 1000) / 1000;
+}
+
+function assertSafeTarget(root, target) {
+  const project = path.resolve(root);
+  const resolved = path.resolve(target);
+  const internal = path.resolve(workDirectory(project));
+
+  if (resolved === project) {
+    throw new Error('cachePath must not point at the project root');
+  }
+  const projectInsideTarget = path.relative(resolved, project);
+  if (
+    projectInsideTarget !== '' &&
+    !projectInsideTarget.startsWith('..') &&
+    !path.isAbsolute(projectInsideTarget)
+  ) {
+    throw new Error(`cachePath ${resolved} contains the project itself`);
+  }
+  if (resolved === internal || resolved.startsWith(`${internal}${path.sep}`)) {
+    throw new Error(`cachePath ${resolved} is inside cache-verdict's own work directory`);
+  }
+}
+
+function resolveTargets(root, config, options = {}) {
+  const resolved = resolvePathPatterns(root, config.cachePath, options);
+  for (const target of resolved.targets) assertSafeTarget(root, target);
+  return resolved;
+}
+
+function moveWorkingTargets(targets, stashRoot) {
+  const moved = [];
+  try {
+    for (let index = 0; index < targets.length; index += 1) {
+      const target = targets[index];
+      if (!pathExists(target)) continue;
+      const stash = path.join(stashRoot, String(index));
+      movePath(target, stash);
+      moved.push({ target, stash });
+    }
+  } catch (error) {
+    for (const entry of [...moved].reverse()) movePath(entry.stash, entry.target);
+    removePath(stashRoot);
+    throw error;
+  }
+  return moved;
+}
+
+function restoreWorkingTargets(root, config, moved, stashRoot) {
+  const produced = resolveTargets(root, config, { existingOnly: true }).targets;
+  const current = new Set([...moved.map((entry) => entry.target), ...produced]);
+  for (const target of current) removePath(target);
+  for (const entry of moved) {
+    if (pathExists(entry.stash)) movePath(entry.stash, entry.target);
+  }
+  removePath(stashRoot);
+}
+
+/**
+ * Take one timed sample. Existing cache directories are moved aside first, a
+ * stored cache for this exact key (if any) is restored, the build runs once,
+ * and the produced artifacts are stored for the next identical key.
+ *
+ * @param {string} root absolute project directory
+ * @param {{ cold?: boolean }} [options]
+ */
+export function runSample(root, options = {}) {
+  const directory = path.resolve(root);
+  const config = loadConfig(directory);
+  const forceCold = options.cold === true;
+  const unresolvedBefore = resolveTargets(directory, config);
+  if (config.cachePath.length > 0 && unresolvedBefore.targets.length === 0) {
+    throw new Error(
+      `cannot resolve any cachePath entries (${unresolvedBefore.unresolved.join(', ') || config.cachePath.join(', ')}); set the referenced environment variables or edit cachePath`,
+    );
+  }
+
+  const { key } = computeInputKey(directory, config);
+  const before = resolveTargets(directory, config, { existingOnly: true }).targets;
+
+  ensureWorkDirectory(directory);
+  const stashRoot = path.join(
+    tempDirectory(directory),
+    `${Date.now()}-${process.pid}-${randomUUID()}`,
+  );
+  const moved = moveWorkingTargets(before, stashRoot);
+
+  let hit = false;
+  let restoreMs = 0;
+  let restoreBytes = 0;
+  let buildMs = 0;
+  let produced = [];
+  let sizeBytes = 0;
+  let stored = false;
+
+  try {
+    if (!forceCold) {
+      const storedCache = readStoredCache(directory, key);
+      if (storedCache !== null) {
+        const restoreStart = nowMs();
+        const restored = restoreStoredCache(directory, storedCache);
+        restoreMs = nowMs() - restoreStart;
+        restoreBytes = restored.restoredBytes;
+        hit = true;
+        touchStoredCache(directory, storedCache);
+      }
+    }
+
+    const buildStart = nowMs();
+    const result = spawnSync(config.command, {
+      cwd: directory,
+      shell: true,
+      stdio: 'inherit',
+    });
+    buildMs = nowMs() - buildStart;
+
+    if (result.error) throw result.error;
+    const status = result.status ?? 1;
+    if (status !== 0) {
+      throw new Error(`build command exited with status ${status}: ${config.command}`);
+    }
+
+    produced = resolveTargets(directory, config, { existingOnly: true }).targets;
+    sizeBytes = produced.reduce((total, target) => total + pathSize(target), 0);
+
+    if (produced.length > 0) {
+      storeCache(directory, key, produced);
+      stored = true;
+    } else if (!forceCold) {
+      // The build used to produce artifacts for this key but no longer does.
+      removePath(storeKeyDirectory(directory, key));
+    }
+
+    const totalMs = restoreMs + buildMs;
+    const sample = {
+      schemaVersion: 1,
+      timestamp: new Date().toISOString(),
+      key,
+      hit,
+      miss: !hit,
+      cold: !hit,
+      warm: hit,
+      temperature: hit ? 'warm' : 'cold',
+      forcedCold: forceCold,
+      buildMs: roundMs(buildMs),
+      restoreMs: roundMs(restoreMs),
+      restoreCostMs: roundMs(restoreMs),
+      totalMs: roundMs(totalMs),
+      sizeBytes,
+      cacheSizeBytes: sizeBytes,
+      restoreSizeBytes: restoreBytes,
+      stored,
+      command: config.command,
+    };
+
+    appendLedgerSample(directory, sample);
+    removePath(stashRoot);
+    return sample;
+  } catch (error) {
+    restoreWorkingTargets(directory, config, moved, stashRoot);
+    throw error;
+  }
+}
+
+export function formatRunSummary(sample, config) {
+  const temperature = sample.hit ? 'HIT (warm)' : 'MISS (cold)';
+  return [
+    `cache-verdict: ${temperature}`,
+    `  key:     ${sample.key}`,
+    `  command: ${config.command}`,
+    `  restore: ${sample.restoreMs.toFixed(1)} ms`,
+    `  build:   ${sample.buildMs.toFixed(1)} ms`,
+    `  total:   ${sample.totalMs.toFixed(1)} ms`,
+    `  size:    ${sample.sizeBytes} bytes${sample.stored ? '' : ' (no cache artifacts stored)'}`,
+  ].join('\n');
+}

package/package.json

@@ -1,6 +1,43 @@
 {
   "name": "cache-verdict",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Times build-cache cold vs warm runs and says whether the cache pays",
+  "license": "Apache-2.0",
+  "keywords": [
+    "build-cache",
+    "cache",
+    "ci",
+    "cold-start",
+    "benchmark",
+    "cargo",
+    "gradle",
+    "nextjs",
+    "npm",
+    "python",
+    "go"
+  ],
+  "type": "module",
+  "bin": {
+    "cache-verdict": "./bin/cache-verdict.mjs"
+  },
+  "files": [
+    "bin",
+    "lib",
+    "examples"
+  ],
+  "scripts": {
+    "test": "node --test",
+    "example": "node examples/run.mjs"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/WJDdidi-1/cache-verdict.git"
+  },
+  "homepage": "https://github.com/WJDdidi-1/cache-verdict#readme",
+  "bugs": {
+    "url": "https://github.com/WJDdidi-1/cache-verdict/issues"
+  }
+}