cache-verdict 0.0.0-stage → 0.1.0

package/bin/cache-verdict.mjs

@@ -0,0 +1,383 @@
+#!/usr/bin/env node
+import { readFileSync, statSync } from 'node:fs';
+import path from 'node:path';
+
+import {
+  detectProject,
+  isSupportedEcosystem,
+  supportedEcosystemIds,
+} from '../lib/detect.mjs';
+import {
+  CONFIG_FILENAME,
+  buildConfig,
+  loadConfig,
+  serializeConfig,
+  writeConfig,
+} from '../lib/config.mjs';
+import { pruneStore, readLedger, storeStats } from '../lib/ledger.mjs';
+import { analyzeSamples, formatReport } from '../lib/report.mjs';
+import { formatRunSummary, runSample } from '../lib/run.mjs';
+
+const VERSION = JSON.parse(
+  readFileSync(new URL('../package.json', import.meta.url), 'utf8'),
+).version;
+
+const USAGE = `cache-verdict: times build-cache cold vs warm runs and says whether the cache pays
+
+Usage:
+  cache-verdict init [directory] [options]
+  cache-verdict run [directory] [--cold]
+  cache-verdict report [directory] [--json]
+  cache-verdict check [directory]
+  cache-verdict prune [directory] --keep <n>
+  cache-verdict --help
+  cache-verdict --version
+
+Commands:
+  init    Detect Cargo, Gradle, Next.js, npm, Python, or Go in a project and
+          write a reviewable .cache-verdict.json beside its build files.
+  run     Move declared cache dirs aside, restore a cache for the content-
+          addressed key when one exists, time the build, store the produced
+          artifacts, and append a sample to .cache-verdict/ledger.jsonl.
+  report  Print hit rate, median cold and warm totals, restore cost, net ms
+          per hit, cache size, and one verdict.
+  check   Exit 0 for cache-pays, 2 for cache-costs, cache-churns,
+          cache-never-hits, or cache-unobserved, and 3 when more runs are
+          needed before a verdict is possible.
+  prune   Evict least-recently-used stored cache keys.
+
+Init options:
+  -e, --ecosystem <id>  Use this detector when several build systems match.
+                        One of: ${supportedEcosystemIds().join(', ')}
+      --command <cmd>   Override the detected build command.
+  -f, --force           Overwrite an existing .cache-verdict.json.
+      --dry-run         Print the config to stdout instead of writing it.
+  -h, --help            Show this help.
+
+Run options:
+      --cold            Force a cold baseline sample. A stored cache is kept
+                        for the next normal run, so it does not prevent a hit.
+  -h, --help            Show this help.
+
+Report options:
+      --json            Print the analysis as JSON.
+  -h, --help            Show this help.
+
+Prune options:
+  -k, --keep <n>        Number of most-recently-used stored keys to retain.
+  -h, --help            Show this help.
+
+The config records the detected build command, the files whose contents make
+up the cache key, the cache paths to measure, and why the project was detected.
+Review and edit it before sampling. Detection order is Next.js, Cargo, Gradle,
+Go, Python, then npm; use --ecosystem to override.
+
+Measured files live under .cache-verdict/ in the project: ledger.jsonl is the
+append-only sample history and store/ holds one content-addressed cache per
+key. Add .cache-verdict/ to .gitignore.
+
+Exit codes:
+  0  success; check found that the cache pays
+  1  no matching build system, unreadable directory, existing config, or a
+     failed build command
+  2  invalid command-line usage; check found a losing, churning, unobserved,
+     or never-hitting cache
+  3  check needs more runs before it can decide
+`;
+
+class UsageError extends Error {}
+
+class CliError extends Error {}
+
+function resolveDirectory(value) {
+  const directory = path.resolve(process.cwd(), value);
+  let stat;
+  try {
+    stat = statSync(directory);
+  } catch {
+    throw new CliError(`cannot read directory ${directory}`);
+  }
+  if (!stat.isDirectory()) throw new CliError(`${directory} is not a directory`);
+  return directory;
+}
+
+function parseInitOptions(args) {
+  const options = {
+    dir: '.',
+    force: false,
+    dryRun: false,
+    ecosystem: null,
+    command: undefined,
+    help: false,
+  };
+  const positional = [];
+
+  for (let index = 0; index < args.length; index += 1) {
+    const argument = args[index];
+
+    if (argument === '--help' || argument === '-h') {
+      options.help = true;
+    } else if (argument === '--force' || argument === '-f') {
+      options.force = true;
+    } else if (argument === '--dry-run') {
+      options.dryRun = true;
+    } else if (argument === '--ecosystem' || argument === '-e') {
+      const value = args[index + 1];
+      if (value === undefined || value.startsWith('-')) {
+        throw new UsageError(`${argument} needs an ecosystem id`);
+      }
+      options.ecosystem = value;
+      index += 1;
+    } else if (argument.startsWith('--ecosystem=')) {
+      const value = argument.slice('--ecosystem='.length);
+      if (!value) throw new UsageError('--ecosystem needs an ecosystem id');
+      options.ecosystem = value;
+    } else if (argument === '--command') {
+      const value = args[index + 1];
+      if (value === undefined) throw new UsageError('--command needs a value');
+      options.command = value;
+      index += 1;
+    } else if (argument.startsWith('--command=')) {
+      const value = argument.slice('--command='.length);
+      if (!value.trim()) throw new UsageError('--command needs a value');
+      options.command = value;
+    } else if (argument === '--') {
+      positional.push(...args.slice(index + 1));
+      break;
+    } else if (argument.startsWith('-')) {
+      throw new UsageError(`unknown option ${argument}`);
+    } else {
+      positional.push(argument);
+    }
+  }
+
+  if (positional.length > 1) {
+    throw new UsageError('init accepts at most one directory');
+  }
+  if (positional.length === 1) options.dir = positional[0];
+  if (options.command !== undefined && !options.command.trim()) {
+    throw new UsageError('--command needs a value');
+  }
+
+  return options;
+}
+
+function parseSimpleOptions(args, knownFlags) {
+  const options = { dir: '.', help: false };
+  for (const flag of knownFlags) options[flag] = false;
+  const positional = [];
+
+  for (let index = 0; index < args.length; index += 1) {
+    const argument = args[index];
+    if (argument === '--help' || argument === '-h') {
+      options.help = true;
+    } else if (argument === '--') {
+      positional.push(...args.slice(index + 1));
+      break;
+    } else if (argument.startsWith('-') && knownFlags.includes(argument.slice(2))) {
+      options[argument.slice(2)] = true;
+    } else if (argument.startsWith('-')) {
+      throw new UsageError(`unknown option ${argument}`);
+    } else {
+      positional.push(argument);
+    }
+  }
+
+  if (positional.length > 1) throw new UsageError('this command accepts at most one directory');
+  if (positional.length === 1) options.dir = positional[0];
+  return options;
+}
+
+function parsePruneOptions(args) {
+  const options = { dir: '.', keep: null, help: false };
+  const positional = [];
+
+  for (let index = 0; index < args.length; index += 1) {
+    const argument = args[index];
+    if (argument === '--help' || argument === '-h') {
+      options.help = true;
+    } else if (argument === '--keep' || argument === '-k') {
+      const value = args[index + 1];
+      if (value === undefined || value.startsWith('-')) {
+        throw new UsageError(`${argument} needs a count`);
+      }
+      options.keep = value;
+      index += 1;
+    } else if (argument.startsWith('--keep=')) {
+      options.keep = argument.slice('--keep='.length);
+    } else if (argument === '--') {
+      positional.push(...args.slice(index + 1));
+      break;
+    } else if (argument.startsWith('-')) {
+      throw new UsageError(`unknown option ${argument}`);
+    } else {
+      positional.push(argument);
+    }
+  }
+
+  if (positional.length > 1) throw new UsageError('prune accepts at most one directory');
+  if (positional.length === 1) options.dir = positional[0];
+  if (options.help) return options;
+
+  if (options.keep === null) throw new UsageError('prune needs --keep <n>');
+  if (!/^\d+$/.test(String(options.keep))) {
+    throw new UsageError('--keep needs a non-negative integer');
+  }
+  options.keep = Number(options.keep);
+  return options;
+}
+
+function init(args) {
+  const options = parseInitOptions(args);
+  if (options.help) {
+    process.stdout.write(USAGE);
+    return 0;
+  }
+
+  if (options.ecosystem !== null && !isSupportedEcosystem(options.ecosystem)) {
+    throw new UsageError(
+      `unsupported ecosystem "${options.ecosystem}"; choose one of: ${supportedEcosystemIds().join(', ')}`,
+    );
+  }
+
+  const directory = resolveDirectory(options.dir);
+  const detection = detectProject(directory, { ecosystem: options.ecosystem });
+  const config = buildConfig(detection, options.command === undefined ? {} : { command: options.command });
+
+  if (options.dryRun) {
+    process.stdout.write(serializeConfig(config));
+    return 0;
+  }
+
+  const target = writeConfig(directory, config, { force: options.force });
+  const shown = path.relative(process.cwd(), target) || CONFIG_FILENAME;
+
+  console.log(`Detected ${detection.displayName} from ${detection.detectedFrom.join(', ')}.`);
+  console.log(`Wrote ${shown}.`);
+  if (detection.alternatives.length > 0) {
+    const alternatives = detection.alternatives.map((item) => item.ecosystem).join(', ');
+    console.log(`Also detected: ${alternatives}. Use --ecosystem to choose one instead.`);
+  }
+  for (const note of config.notes ?? []) console.log(`note: ${note}`);
+  console.log('Review "command", "inputs", and "cachePath" before sampling.');
+
+  return 0;
+}
+
+function run(args) {
+  const options = parseSimpleOptions(args, ['cold']);
+  if (options.help) {
+    process.stdout.write(USAGE);
+    return 0;
+  }
+
+  const directory = resolveDirectory(options.dir);
+  const config = loadConfig(directory);
+  const sample = runSample(directory, { cold: options.cold });
+  console.log(formatRunSummary(sample, config));
+  return 0;
+}
+
+function analyzeProject(directory) {
+  const config = loadConfig(directory);
+  const samples = readLedger(directory);
+  const stats = storeStats(directory);
+  return analyzeSamples(samples, {
+    hasCachePaths: config.cachePath.length > 0,
+    storedKeys: stats.keys,
+    storedSizeBytes: stats.sizeBytes,
+  });
+}
+
+function report(args) {
+  const options = parseSimpleOptions(args, ['json']);
+  if (options.help) {
+    process.stdout.write(USAGE);
+    return 0;
+  }
+
+  const directory = resolveDirectory(options.dir);
+  const analysis = analyzeProject(directory);
+  if (options.json) {
+    console.log(JSON.stringify(analysis, null, 2));
+  } else {
+    process.stdout.write(formatReport(analysis));
+  }
+  return 0;
+}
+
+function check(args) {
+  const options = parseSimpleOptions(args, []);
+  if (options.help) {
+    process.stdout.write(USAGE);
+    return 0;
+  }
+
+  const directory = resolveDirectory(options.dir);
+  const analysis = analyzeProject(directory);
+  process.stdout.write(formatReport(analysis));
+
+  if (analysis.moreRunsNeeded) {
+    process.stderr.write(
+      'cache-verdict: not enough samples yet; run the build again before gating CI\n',
+    );
+    return 3;
+  }
+  if (analysis.verdict === 'cache-pays') return 0;
+
+  process.stderr.write(`cache-verdict: ${analysis.verdict}; the cache did not pay for itself\n`);
+  return 2;
+}
+
+function prune(args) {
+  const options = parsePruneOptions(args);
+  if (options.help) {
+    process.stdout.write(USAGE);
+    return 0;
+  }
+
+  const directory = resolveDirectory(options.dir);
+  loadConfig(directory);
+  const result = pruneStore(directory, options.keep);
+  console.log(
+    `Kept ${result.kept.length} stored key${result.kept.length === 1 ? '' : 's'}; evicted ${result.evicted.length}.`,
+  );
+  return 0;
+}
+
+function main(argv) {
+  if (argv.length === 0) {
+    process.stdout.write(USAGE);
+    return 0;
+  }
+
+  const [command, ...rest] = argv;
+
+  if (command === '--help' || command === '-h' || command === 'help') {
+    process.stdout.write(USAGE);
+    return 0;
+  }
+  if (command === '--version' || command === '-v') {
+    console.log(VERSION);
+    return 0;
+  }
+  if (command === 'init') return init(rest);
+  if (command === 'run') return run(rest);
+  if (command === 'report') return report(rest);
+  if (command === 'check') return check(rest);
+  if (command === 'prune') return prune(rest);
+  if (command.startsWith('-')) throw new UsageError(`unknown option ${command}`);
+  throw new UsageError(`unknown command "${command}"`);
+}
+
+try {
+  process.exitCode = main(process.argv.slice(2));
+} catch (error) {
+  if (error instanceof UsageError) {
+    process.stderr.write(`cache-verdict: ${error.message}\n\n${USAGE}`);
+    process.exitCode = 2;
+  } else {
+    process.stderr.write(`cache-verdict: ${error.message}\n`);
+    process.exitCode = Number.isInteger(error.exitCode) ? error.exitCode : 1;
+  }
+}

package/examples/npm-build-cache/README.md

@@ -0,0 +1,95 @@
+# npm build-cache example
+
+A tiny npm project with a deliberately slow cold build and a fast warm build.
+`cache-verdict` can show the difference in numbers instead of a hit counter.
+
+- `src/input.txt` is the input. Editing it changes the cache key.
+- `build.mjs` reads `src/input.txt`, writes an artifact to
+  `node_modules/.cache/`, and spends 700 ms "compiling" only when that artifact
+  is missing. The cache path `node_modules/.cache` is what `cache-verdict`
+  moves aside, restores, sizes, and stores.
+- `package.json` provides the `npm run build` command that `init` detects.
+
+## Run it from the repository root
+
+```sh
+npm run example
+```
+
+That copies this directory to a temporary directory, runs `init`, two `run`
+samples, and `report` with the checkout's CLI, then checks that the verdict is
+`cache-pays`. The temporary copy is removed afterwards, so nothing is written
+into the repository.
+
+## Run it by hand
+
+From this directory:
+
+```sh
+node ../../bin/cache-verdict.mjs init
+node ../../bin/cache-verdict.mjs run
+node ../../bin/cache-verdict.mjs run
+node ../../bin/cache-verdict.mjs report
+```
+
+The first `run` is a cold miss and does the slow work. The second restores the
+stored cache, and the report should say `cache-pays`. `.cache-verdict/` and
+`dist/` are ignored by Git; delete them to start over.
+
+Expected shape of the output (timings vary by machine):
+
+```console
+$ node ../../bin/cache-verdict.mjs run
+...
+cache-verdict: MISS (cold)
+...
+$ node ../../bin/cache-verdict.mjs run
+...
+cache-verdict: HIT (warm)
+...
+$ node ../../bin/cache-verdict.mjs report
+Runs: 2 (1 hit, 1 miss)
+...
+Verdict: cache-pays
+```
+
+If the report says `cache-costs` or `cache-churns`, the cache is not paying for
+itself for this key; that is the decision the tool exists to make. To see a
+churn, edit `src/input.txt` between runs and `run` again: the key changes, so
+the next build is cold.
+
+## What `init` writes
+
+`init` detects npm from `package.json` and writes this reviewable config:
+
+```json
+{
+  "schemaVersion": 1,
+  "ecosystem": "npm",
+  "displayName": "npm",
+  "command": "npm run build",
+  "inputs": [
+    "package.json",
+    "*.js",
+    "*.mjs",
+    "*.cjs",
+    "*.ts",
+    "*.tsx",
+    "*.jsx",
+    "src/**",
+    "lib/**",
+    "bin/**",
+    "scripts/**"
+  ],
+  "cachePath": ["node_modules/.cache"],
+  "detectedFrom": ["package.json"],
+  "notes": [
+    "npm's download cache defaults to ~/.npm; set npm_config_cache=.npm in CI to keep it project-local and measurable."
+  ]
+}
+```
+
+The note is about npm's package-download cache, which this example does not
+measure. The measured cache is `node_modules/.cache`, written by `build.mjs`.
+Edit `command`, `inputs`, or `cachePath` if the detection does not match the
+build you care about, then sample again with a fresh directory.

package/examples/npm-build-cache/build.mjs

@@ -0,0 +1,25 @@
+// A deliberately simple build: derive an artifact from src/input.txt, keep it
+// in node_modules/.cache, and spend 700 ms of "compile" time only on a cold
+// cache. That gap is what cache-verdict measures.
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+
+const source = readFileSync('src/input.txt', 'utf8');
+const digest = createHash('sha256').update(source).digest('hex').slice(0, 12);
+const cached = `node_modules/.cache/${digest}.txt`;
+
+if (existsSync(cached)) {
+  console.log(`warm build: reusing cached artifact ${digest}`);
+} else {
+  console.log(`cold build: computing ${digest} (this is the slow part)`);
+  const until = Date.now() + 700;
+  while (Date.now() < until) {
+    // Stand-in for real compilation work.
+  }
+  mkdirSync('node_modules/.cache', { recursive: true });
+  writeFileSync(cached, `${digest}: ${source.trim().length} characters of input\n`);
+}
+
+mkdirSync('dist', { recursive: true });
+writeFileSync('dist/result.txt', readFileSync(cached));
+console.log('wrote dist/result.txt');

package/examples/npm-build-cache/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "cache-verdict-example-npm-build-cache",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Runnable cache-verdict example: a slow cold build that is fast when the cache is restored",
+  "type": "module",
+  "scripts": {
+    "build": "node build.mjs"
+  }
+}

package/examples/npm-build-cache/src/input.txt

@@ -0,0 +1,5 @@
+cache-verdict example input
+
+This file is an input to the cache key. Editing it produces a new key,
+which makes the next build cold again instead of restoring an artifact
+for different content.

package/examples/run.mjs

@@ -0,0 +1,63 @@
+#!/usr/bin/env node
+// Run the npm-build-cache example against a temporary copy of the project, so
+// trying it out never leaves .cache-verdict/, node_modules/.cache, or dist/ in
+// the checkout. Uses only Node.js built-ins.
+import { spawnSync } from 'node:child_process';
+import { cpSync, mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const cli = path.resolve(here, '..', 'bin', 'cache-verdict.mjs');
+const source = path.join(here, 'npm-build-cache');
+
+const temporary = mkdtempSync(path.join(tmpdir(), 'cache-verdict-example-'));
+const project = path.join(temporary, 'npm-build-cache');
+
+function run(args) {
+  console.log(`$ node ${path.relative(process.cwd(), cli) || cli} ${args.join(' ')}`);
+  const result = spawnSync(process.execPath, [cli, ...args], {
+    cwd: project,
+    stdio: 'inherit',
+  });
+  if (result.error) throw result.error;
+  if (result.status !== 0) {
+    throw new Error(`cache-verdict ${args.join(' ')} exited with status ${result.status}`);
+  }
+}
+
+try {
+  cpSync(source, project, { recursive: true });
+
+  // A manual run in the checkout may have left state behind; always start clean.
+  for (const leftover of ['node_modules', '.cache-verdict', 'dist']) {
+    rmSync(path.join(project, leftover), { recursive: true, force: true });
+  }
+
+  console.log(`Running the example in a temporary copy: ${project}\n`);
+  run(['init']);
+  run(['run']);
+  run(['run']);
+  run(['report']);
+
+  // check prints the same report again, so only surface it when it disagrees.
+  const check = spawnSync(process.execPath, [cli, 'check'], {
+    cwd: project,
+    encoding: 'utf8',
+  });
+  if (check.error) throw check.error;
+  if (check.status !== 0) {
+    process.stderr.write(check.stdout ?? '');
+    process.stderr.write(check.stderr ?? '');
+    throw new Error(`cache-verdict check exited with status ${check.status}`);
+  }
+
+  console.log('\ncheck exited 0: this example cache pays.');
+} catch (error) {
+  process.exitCode = 1;
+  console.error(`\nexample failed: ${error.message}`);
+} finally {
+  rmSync(temporary, { recursive: true, force: true });
+  console.log('Removed the temporary copy.');
+}