cache-verdict 0.0.0-stage → 0.1.0

package/lib/glob.mjs

@@ -0,0 +1,192 @@
+import { lstatSync, readdirSync } from 'node:fs';
+import path from 'node:path';
+
+/**
+ * Paths in configs and ledgers always use forward slashes so a config written
+ * on Windows still describes the same inputs on Linux CI.
+ */
+export function toPosix(value) {
+  return value.split(path.sep).join('/');
+}
+
+export function hasGlob(value) {
+  return /[*?[\]{}]/.test(value);
+}
+
+export function expandEnvironment(value, environment = process.env) {
+  return value.replace(/\$(?:\{([A-Za-z_][A-Za-z0-9_]*)\}|([A-Za-z_][A-Za-z0-9_]*))/g, (match, braced, bare) => {
+    const name = braced ?? bare;
+    return Object.prototype.hasOwnProperty.call(environment, name) ? environment[name] : match;
+  });
+}
+
+function escapeRegExp(character) {
+  return /[\\^$.*+?()[\]{}|]/.test(character) ? `\\${character}` : character;
+}
+
+/**
+ * Translate the small glob dialect used by detectors into a regular
+ * expression: `*` stays inside one path segment, `**` crosses segments, and
+ * a double-star directory prefix also matches zero directories.
+ */
+export function globToRegExp(pattern) {
+  let source = '';
+  for (let index = 0; index < pattern.length; index += 1) {
+    const character = pattern[index];
+
+    if (character === '*') {
+      if (pattern[index + 1] === '*') {
+        while (pattern[index + 1] === '*') index += 1;
+        if (pattern[index + 1] === '/') {
+          index += 1;
+          source += '(?:.*/)?';
+        } else {
+          source += '.*';
+        }
+      } else {
+        source += '[^/]*';
+      }
+    } else if (character === '?') {
+      source += '[^/]';
+    } else {
+      source += escapeRegExp(character);
+    }
+  }
+  return new RegExp(`^${source}$`);
+}
+
+export function matchGlob(relativePath, pattern) {
+  return globToRegExp(toPosix(pattern)).test(toPosix(relativePath));
+}
+
+function lstatSafe(target) {
+  try {
+    return lstatSync(target);
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Walk a project without following symlinks. Generated dependency trees are
+ * skipped by default because they are never useful build inputs and can make
+ * content hashing unexpectedly expensive.
+ */
+export function walkTree(root, options = {}) {
+  const skipDirectories = new Set(
+    options.skipDirectories ?? ['.git', '.hg', '.svn', '.cache-verdict', 'node_modules'],
+  );
+  const files = [];
+  const directories = [];
+
+  function walk(relativeDirectory) {
+    const absoluteDirectory =
+      relativeDirectory === ''
+        ? root
+        : path.join(root, ...relativeDirectory.split('/').filter(Boolean));
+
+    let entries;
+    try {
+      entries = readdirSync(absoluteDirectory, { withFileTypes: true });
+    } catch {
+      return;
+    }
+
+    for (const entry of entries) {
+      if (entry.isSymbolicLink()) continue;
+
+      const relative = relativeDirectory === '' ? entry.name : `${relativeDirectory}/${entry.name}`;
+
+      if (entry.isDirectory()) {
+        directories.push(relative);
+        if (skipDirectories.has(entry.name)) continue;
+        walk(relative);
+      } else if (entry.isFile()) {
+        files.push(relative);
+      }
+    }
+  }
+
+  walk('');
+  files.sort();
+  directories.sort();
+  return { files, directories };
+}
+
+function staticPrefix(pattern) {
+  const segments = pattern.split('/');
+  const kept = [];
+  for (const segment of segments) {
+    if (hasGlob(segment)) break;
+    kept.push(segment);
+  }
+  // The final literal segment may be the file itself, so only walk its parent.
+  if (kept.length === segments.length) kept.pop();
+  return kept.join('/');
+}
+
+/**
+ * Resolve declared input or cache paths to absolute filesystem targets.
+ *
+ * A non-glob path is always returned (unless `existingOnly` is set), because
+ * `run` needs to know where a cache directory will appear after the build.
+ * Globs only return paths that currently exist.
+ *
+ * @param {string} root absolute project directory
+ * @param {string[]} patterns declared paths, possibly containing $ENV or globs
+ * @param {{ existingOnly?: boolean, environment?: NodeJS.ProcessEnv }} [options]
+ * @returns {{ targets: string[], unresolved: string[] }}
+ */
+export function resolvePathPatterns(root, patterns, options = {}) {
+  const existingOnly = options.existingOnly === true;
+  const environment = options.environment ?? process.env;
+  const targets = new Set();
+  const unresolved = [];
+
+  for (const declared of patterns) {
+    if (typeof declared !== 'string' || declared.trim() === '') continue;
+
+    const expanded = expandEnvironment(declared.trim(), environment);
+    if (expanded.includes('$')) {
+      unresolved.push(declared);
+      continue;
+    }
+
+    if (path.isAbsolute(expanded)) {
+      if (hasGlob(expanded)) {
+        unresolved.push(declared);
+        continue;
+      }
+      const stat = lstatSafe(expanded);
+      if (!existingOnly || stat !== undefined) targets.add(path.normalize(expanded));
+      continue;
+    }
+
+    if (!hasGlob(expanded)) {
+      const target = path.resolve(root, expanded);
+      const stat = lstatSafe(target);
+      if (!existingOnly || stat !== undefined) targets.add(target);
+      continue;
+    }
+
+    const prefix = staticPrefix(toPosix(expanded));
+    const base = prefix === '' ? root : path.resolve(root, ...prefix.split('/'));
+    const baseStat = lstatSafe(base);
+    if (baseStat === undefined || !baseStat.isDirectory()) continue;
+
+    const pattern = toPosix(expanded);
+    const relativeBase = path.relative(root, base);
+    const start = relativeBase === '' ? '' : toPosix(relativeBase);
+    const tree = walkTree(root, { skipDirectories: ['.git', '.hg', '.svn', '.cache-verdict', 'node_modules'] });
+
+    for (const relative of [...tree.files, ...tree.directories]) {
+      if (start !== '' && !(relative === start || relative.startsWith(`${start}/`))) continue;
+      if (matchGlob(relative, pattern)) targets.add(path.resolve(root, ...relative.split('/')));
+    }
+  }
+
+  return {
+    targets: [...targets].sort(),
+    unresolved,
+  };
+}

package/lib/key.mjs

@@ -0,0 +1,52 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+
+import { matchGlob, toPosix, walkTree } from './glob.mjs';
+
+function sha256(data) {
+  return createHash('sha256').update(data).digest('hex');
+}
+
+/**
+ * Content-address the declared inputs. The command, cache paths, and input
+ * patterns are part of the address too: changing how a build is invoked or
+ * what it measures must not silently reuse an unrelated stored cache.
+ *
+ * @param {string} directory absolute project directory
+ * @param {{ command: string, inputs: string[], cachePath: string[] }} config
+ * @returns {{ key: string, files: Array<{ path: string, sha256: string }> }}
+ */
+export function computeInputKey(directory, config) {
+  const patterns = [...new Set(config.inputs.map(toPosix))].sort();
+  const cachePatterns = [...new Set(config.cachePath.map(toPosix))].sort();
+  const { files } = walkTree(directory);
+
+  const matched = files.filter((relative) =>
+    patterns.some((pattern) => matchGlob(relative, pattern)),
+  );
+
+  const entries = matched.map((relative) => {
+    const absolute = path.join(directory, ...relative.split('/'));
+    let contents;
+    try {
+      contents = readFileSync(absolute);
+    } catch (error) {
+      throw new Error(`cannot read declared input ${relative}: ${error.message}`);
+    }
+    return { path: relative, sha256: sha256(contents) };
+  });
+
+  const payload = {
+    schemaVersion: 1,
+    command: config.command,
+    inputs: patterns,
+    cachePath: cachePatterns,
+    files: entries,
+  };
+
+  return {
+    key: `sha256:${sha256(JSON.stringify(payload))}`,
+    files: entries,
+  };
+}

package/lib/ledger.mjs

@@ -0,0 +1,357 @@
+import { randomUUID } from 'node:crypto';
+import {
+  appendFileSync,
+  cpSync,
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import path from 'node:path';
+
+import { toPosix } from './glob.mjs';
+
+export const WORK_DIRECTORY = '.cache-verdict';
+export const LEDGER_FILENAME = 'ledger.jsonl';
+export const STORE_DIRECTORY = 'store';
+export const MANIFEST_FILENAME = 'manifest.json';
+export const TMP_DIRECTORY = 'tmp';
+
+export function workDirectory(root) {
+  return path.join(root, WORK_DIRECTORY);
+}
+
+export function ledgerPath(root) {
+  return path.join(workDirectory(root), LEDGER_FILENAME);
+}
+
+export function storeDirectory(root) {
+  return path.join(workDirectory(root), STORE_DIRECTORY);
+}
+
+export function tempDirectory(root) {
+  return path.join(workDirectory(root), TMP_DIRECTORY);
+}
+
+export function ensureWorkDirectory(root) {
+  const directory = workDirectory(root);
+  mkdirSync(directory, { recursive: true });
+  return directory;
+}
+
+export function pathExists(target) {
+  return existsSync(target);
+}
+
+export function pathSize(target) {
+  let stat;
+  try {
+    stat = lstatSync(target);
+  } catch {
+    return 0;
+  }
+
+  if (stat.isSymbolicLink()) return stat.size;
+  if (!stat.isDirectory()) return stat.size;
+
+  let total = 0;
+  let entries;
+  try {
+    entries = readdirSync(target, { withFileTypes: true });
+  } catch {
+    return total;
+  }
+  for (const entry of entries) {
+    total += pathSize(path.join(target, entry.name));
+  }
+  return total;
+}
+
+export function removePath(target) {
+  rmSync(target, { recursive: true, force: true });
+}
+
+export function copyPath(source, target) {
+  mkdirSync(path.dirname(target), { recursive: true });
+  cpSync(source, target, {
+    recursive: true,
+    force: true,
+    preserveTimestamps: true,
+    dereference: false,
+  });
+}
+
+/**
+ * Move a cache path out of the build directory. Falling back to copy+delete
+ * keeps absolute cache paths on another filesystem working too.
+ */
+export function movePath(source, target) {
+  mkdirSync(path.dirname(target), { recursive: true });
+  removePath(target);
+  try {
+    renameSync(source, target);
+  } catch (error) {
+    if (error.code !== 'EXDEV') throw error;
+    copyPath(source, target);
+    removePath(source);
+  }
+}
+
+export function readLedger(root) {
+  const target = ledgerPath(root);
+  let text;
+  try {
+    text = readFileSync(target, 'utf8');
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw new Error(`cannot read ${target}: ${error.message}`);
+  }
+
+  const samples = [];
+  const lines = text.split(/\r?\n/);
+  for (let index = 0; index < lines.length; index += 1) {
+    const line = lines[index].trim();
+    if (line === '') continue;
+    try {
+      const sample = JSON.parse(line);
+      if (sample === null || typeof sample !== 'object' || Array.isArray(sample)) {
+        throw new Error('sample must be a JSON object');
+      }
+      samples.push(sample);
+    } catch (error) {
+      throw new Error(`cannot parse ${target} line ${index + 1}: ${error.message}`);
+    }
+  }
+  return samples;
+}
+
+export function appendLedgerSample(root, sample) {
+  ensureWorkDirectory(root);
+  appendFileSync(ledgerPath(root), `${JSON.stringify(sample)}\n`, 'utf8');
+}
+
+function keySegment(key) {
+  const match = /^sha256:([a-f0-9]{64})$/.exec(key);
+  if (!match) throw new Error(`invalid cache key ${JSON.stringify(key)}`);
+  return match[1];
+}
+
+export function storeKeyDirectory(root, key) {
+  return path.join(storeDirectory(root), keySegment(key));
+}
+
+export function encodeStoredTarget(root, target) {
+  const relative = path.relative(root, target);
+  if (relative !== '' && !relative.startsWith('..') && !path.isAbsolute(relative)) {
+    return { kind: 'relative', path: toPosix(relative) };
+  }
+  return { kind: 'absolute', path: target };
+}
+
+export function decodeStoredTarget(root, target) {
+  if (target?.kind === 'relative' && typeof target.path === 'string') {
+    return path.resolve(root, ...target.path.split('/'));
+  }
+  if (target?.kind === 'absolute' && typeof target.path === 'string') {
+    return path.normalize(target.path);
+  }
+  throw new Error('stored cache manifest contains an invalid target');
+}
+
+function readManifestFile(directory) {
+  const target = path.join(directory, MANIFEST_FILENAME);
+  let text;
+  try {
+    text = readFileSync(target, 'utf8');
+  } catch (error) {
+    if (error.code === 'ENOENT') return null;
+    throw new Error(`cannot read ${target}: ${error.message}`);
+  }
+
+  let manifest;
+  try {
+    manifest = JSON.parse(text);
+  } catch (error) {
+    throw new Error(`cannot parse ${target}: ${error.message}`);
+  }
+
+  if (manifest === null || typeof manifest !== 'object' || !Array.isArray(manifest.slots)) {
+    throw new Error(`${target} must contain a manifest object with "slots"`);
+  }
+  return manifest;
+}
+
+export function readStoredCache(root, key) {
+  const directory = storeKeyDirectory(root, key);
+  const manifest = readManifestFile(directory);
+  if (manifest === null) return null;
+  if (manifest.key !== key) {
+    throw new Error(`stored cache ${key} has a mismatched manifest`);
+  }
+
+  const slots = manifest.slots
+    .filter((slot) => Number.isInteger(slot.index) && slot.target !== undefined)
+    .filter((slot) => existsSync(path.join(directory, String(slot.index))))
+    .sort((left, right) => left.index - right.index);
+
+  if (slots.length === 0) return null;
+  return { ...manifest, slots, directory };
+}
+
+export function restoreStoredCache(root, manifest) {
+  let restoredBytes = 0;
+
+  for (const slot of manifest.slots) {
+    const target = decodeStoredTarget(root, slot.target);
+    const source = path.join(manifest.directory, String(slot.index));
+    removePath(target);
+    copyPath(source, target);
+    restoredBytes += Number.isFinite(slot.sizeBytes) ? slot.sizeBytes : pathSize(source);
+  }
+
+  return { restoredBytes, restoredSlots: manifest.slots.length };
+}
+
+export function touchStoredCache(root, manifest) {
+  const { directory, ...stored } = manifest;
+  const updated = { ...stored, lastUsedAt: new Date().toISOString() };
+  writeFileSync(
+    path.join(directory, MANIFEST_FILENAME),
+    `${JSON.stringify(updated, null, 2)}\n`,
+    'utf8',
+  );
+}
+
+/**
+ * Store the artifacts produced by a build under their content-addressed key.
+ * The new directory is built beside the old one and swapped in at the end, so
+ * a failed copy cannot destroy the cache that was just used for a hit.
+ */
+export function storeCache(root, key, targets) {
+  const store = storeDirectory(root);
+  mkdirSync(store, { recursive: true });
+
+  const segment = keySegment(key);
+  const finalDirectory = path.join(store, segment);
+  const staging = path.join(store, `.${segment}.${process.pid}.${randomUUID()}.tmp`);
+  mkdirSync(staging, { recursive: true });
+
+  let sizeBytes = 0;
+  const slots = [];
+  const sorted = [...targets].sort();
+
+  try {
+    for (let index = 0; index < sorted.length; index += 1) {
+      const target = sorted[index];
+      const slotDirectory = path.join(staging, String(index));
+      copyPath(target, slotDirectory);
+      const slotSize = pathSize(target);
+      sizeBytes += slotSize;
+      slots.push({
+        index,
+        target: encodeStoredTarget(root, target),
+        sizeBytes: slotSize,
+      });
+    }
+
+    const now = new Date().toISOString();
+    const manifest = {
+      schemaVersion: 1,
+      key,
+      createdAt: now,
+      lastUsedAt: now,
+      sizeBytes,
+      slots,
+    };
+    writeFileSync(
+      path.join(staging, MANIFEST_FILENAME),
+      `${JSON.stringify(manifest, null, 2)}\n`,
+      'utf8',
+    );
+
+    removePath(finalDirectory);
+    renameSync(staging, finalDirectory);
+    return manifest;
+  } catch (error) {
+    removePath(staging);
+    throw error;
+  }
+}
+
+function listStoreDirectories(root) {
+  const store = storeDirectory(root);
+  let entries;
+  try {
+    entries = readdirSync(store, { withFileTypes: true });
+  } catch (error) {
+    if (error.code === 'ENOENT') return [];
+    throw new Error(`cannot read ${store}: ${error.message}`);
+  }
+
+  return entries
+    .filter((entry) => entry.isDirectory() && !entry.name.startsWith('.'))
+    .map((entry) => path.join(store, entry.name));
+}
+
+export function storeStats(root) {
+  const directories = listStoreDirectories(root);
+  let sizeBytes = 0;
+  let keys = 0;
+
+  for (const directory of directories) {
+    const manifest = readManifestFile(directory);
+    if (manifest === null) {
+      sizeBytes += pathSize(directory);
+      keys += 1;
+      continue;
+    }
+    sizeBytes += Number.isFinite(manifest.sizeBytes) ? manifest.sizeBytes : pathSize(directory);
+    keys += 1;
+  }
+
+  return { keys, sizeBytes };
+}
+
+/**
+ * Evict least-recently-used stored keys. A hit updates lastUsedAt, so pruning
+ * keeps the keys the sampled builds actually restore.
+ */
+export function pruneStore(root, keep) {
+  const directories = listStoreDirectories(root);
+  const entries = directories.map((directory) => {
+    let manifest = null;
+    try {
+      manifest = readManifestFile(directory);
+    } catch {
+      manifest = null;
+    }
+    const lastUsed = manifest?.lastUsedAt ? Date.parse(manifest.lastUsedAt) : Number.NaN;
+    let fallback = 0;
+    try {
+      fallback = statSync(directory).mtimeMs;
+    } catch {
+      fallback = 0;
+    }
+    return {
+      directory,
+      key: manifest?.key ?? path.basename(directory),
+      lastUsedAt: Number.isFinite(lastUsed) ? lastUsed : fallback,
+    };
+  });
+
+  entries.sort((left, right) => right.lastUsedAt - left.lastUsedAt);
+
+  const kept = entries.slice(0, keep);
+  const evicted = entries.slice(keep);
+  for (const entry of evicted) removePath(entry.directory);
+
+  return {
+    kept: kept.map((entry) => entry.key),
+    evicted: evicted.map((entry) => entry.key),
+  };
+}