@williamthorsen/nmr 0.48.0 → 0.50.0

package/dist/esm/check-cache-entries.js

@@ -0,0 +1,87 @@
+import path from 'node:path';
+import { readCacheEntry, readJsonCacheEntry, removeCacheDir, removeCacheEntry, resolveCacheEntryPath, writeCacheEntry, } from '@williamthorsen/nmr-core';
+import { describeError } from '@williamthorsen/toolbelt.errors';
+import { writeDebugNote } from "./check-cache.js";
+import { isObject, isStringRecord } from "./helpers/type-guards.js";
+const CACHE_TOOL = 'nmr-check';
+const UNSAFE_SLUG_CHARACTERS = /[^\w.-]+/g;
+export async function certifyRetention(options) {
+    const { entry, runId } = options;
+    if (entry.retention === undefined || entry.retention.runId === runId) {
+        return;
+    }
+    try {
+        await writeCheckCacheEntry({
+            anchorDir: options.anchorDir,
+            command: options.command,
+            monorepoRoot: options.monorepoRoot,
+            entry: { ...entry, retention: { ...entry.retention, runId } },
+        });
+    }
+    catch (error) {
+        writeDebugNote(`could not certify ${options.command}: ${describeError(error)}`, options.env, options.stderr);
+    }
+}
+export async function readCheckCacheEntry(options) {
+    const parsedEntry = await readJsonCacheEntry(resolveEntryPath(options), isParsedCheckCacheEntry);
+    if (parsedEntry === undefined) {
+        return undefined;
+    }
+    const { retention, ...pass } = parsedEntry;
+    return isRetention(retention) ? { ...pass, retention } : pass;
+}
+export async function readTranscript(ref) {
+    return readCacheEntry(resolveEntryPath(ref, '.log'));
+}
+export async function recordTranscript(ref, transcript) {
+    const entryPath = resolveEntryPath(ref, '.log');
+    if (transcript === undefined) {
+        await removeCacheEntry(entryPath);
+        return;
+    }
+    await writeCacheEntry(entryPath, transcript);
+}
+export async function removeCheckCache(scopeDir) {
+    await removeCacheDir({ tool: CACHE_TOOL, scopeDir });
+}
+export function isReplayLine(value) {
+    return (isObject(value) &&
+        typeof value['command'] === 'string' &&
+        typeof value['excerpt'] === 'string' &&
+        typeof value['scope'] === 'string');
+}
+export async function writeCheckCacheEntry(options) {
+    await writeCacheEntry(resolveEntryPath(options), JSON.stringify(options.entry, undefined, 2));
+}
+function isParsedCheckCacheEntry(value) {
+    if (!isObject(value)) {
+        return false;
+    }
+    const stringFields = ['key', 'treeHash', 'headSha', 'commandString', 'nmrVersion', 'nodeVersion', 'recordedAt'];
+    if (stringFields.some((field) => typeof value[field] !== 'string')) {
+        return false;
+    }
+    return ((value['host'] === undefined || typeof value['host'] === 'string') &&
+        typeof value['durationMs'] === 'number' &&
+        Number.isFinite(value['durationMs']) &&
+        !Number.isNaN(Date.parse(String(value['recordedAt']))) &&
+        isStringRecord(value['buildDigests']));
+}
+function isRetention(value) {
+    return (isObject(value) &&
+        typeof value['key'] === 'string' &&
+        typeof value['runId'] === 'string' &&
+        Array.isArray(value['replay']) &&
+        value['replay'].every((line) => isReplayLine(line)));
+}
+function resolveEntryPath(options, extension = '.json') {
+    const anchorDir = path.resolve(options.anchorDir);
+    const ref = {
+        tool: CACHE_TOOL,
+        scopeDir: options.monorepoRoot,
+        slug: `${path.basename(anchorDir)}-${options.command.replaceAll(UNSAFE_SLUG_CHARACTERS, '-')}`,
+        extension,
+        discriminators: [anchorDir, options.command],
+    };
+    return resolveCacheEntryPath(ref);
+}

package/dist/esm/check-cache-gate.d.ts

@@ -0,0 +1,64 @@
+import type { Writable } from 'node:stream';
+import type { OutputStyle } from '@williamthorsen/nmr-core';
+import { type TreeSnapshot } from './check-cache.js';
+import { type RunPlan } from './invocation.js';
+import type { NmrConfig } from './types.js';
+import type { VerdictOutcome } from './verdict.js';
+/**
+ * Decides whether the check-result cache covers this invocation, and takes the tree snapshot on which it would
+ * gate.
+ * Returns `undefined` when the cache does not apply, which always means the command runs.
+ *
+ * Decided before anything is resolved or spawned, so that a delegating invocation hands the snapshot to its
+ * children rather than leaving each of them to hash the tree again. A hook leaf is out of scope because it is
+ * not a command that anyone asks for: it runs as part of the chain that the gate already covers. Arguments are
+ * out of scope because they change what a command does in ways that the gate cannot read.
+ */
+export declare function openGate(options: {
+    command: string;
+    config: NmrConfig;
+    env: NodeJS.ProcessEnv;
+    monorepoRoot: string;
+    passthrough: string[];
+    stderr: Writable;
+    style: OutputStyle;
+}): TreeSnapshot | undefined;
+/**
+ * Computes the key under which this invocation would be recorded, or `undefined` when the cache does not apply.
+ * The cache does not apply to a `devBin` substitution: the substitute is built from a location that the tree id
+ * does not describe, so a pass by it is not a pass by the command that the key names.
+ */
+export declare function resolveCacheKey(options: {
+    anchorDir: string;
+    command: string;
+    commandString: string;
+    env: NodeJS.ProcessEnv;
+    monorepoRoot: string;
+    snapshot: TreeSnapshot | undefined;
+    stderr: Writable;
+    substitution: string | undefined;
+}): string | undefined;
+/**
+ * Runs the resolved steps behind the check-result cache: skips them when a recorded pass covers this
+ * invocation, and records a pass when one is earned. Returns the exit code and the outcome that a verdict reports,
+ * either way, with the host that earned the outcome when it was not this machine. The outcome is absent when the
+ * invocation already reported why it did not run.
+ *
+ * The cache is consulted before the plan's target is: a miss runs on the host to which the invocation is delegated,
+ * outside this machine's lock, and runs here only when there is not a host to run on.
+ *
+ * The override notice waits until the command is going to run, because naming the script that stands in for a
+ * built-in says nothing useful about an invocation that skipped it.
+ *
+ * Two step lists: `steps` is the chain that runs, the hooks wrapped around the command included, and
+ * `ownSteps` is the command's alone, which is what a composite's assembly reads. A hook contributes nothing to
+ * what a skip replays, as it contributes nothing to a leaf's excerpt.
+ */
+export declare function runGated(plan: RunPlan): Promise<GatedResult>;
+/** How a gated invocation ended, and the host that ran it when it was not this machine. */
+interface GatedResult {
+    exitCode: number;
+    host?: string;
+    outcome?: VerdictOutcome;
+}
+export {};

package/dist/esm/check-cache-gate.js

@@ -0,0 +1,226 @@
+import { formatMisplacedNoCacheWarning, isCacheableCommand, resolveTreeSnapshot, writeDebugNote, } from "./check-cache.js";
+import { findStaleBuildOutput, readBuildOutputState } from "./check-cache-build-output.js";
+import { recordDelegatedPass } from "./check-cache-delegation.js";
+import { certifyRetention, readCheckCacheEntry } from "./check-cache-entries.js";
+import { computeCacheKey, computeRetentionKey } from "./check-cache-key.js";
+import { recordPass } from "./check-cache-record.js";
+import { addLockEnv, takeMachineLock } from "./child-env.js";
+import { runPlanOnHost } from "./delegate-host.js";
+import { VERSION } from "./invocation.js";
+import { resolveChannel } from "./output-channel.js";
+import { writePassManifestEntry } from "./pass-manifest.js";
+import { runSteps } from "./runner.js";
+export function openGate(options) {
+    const { command, config, env, monorepoRoot, passthrough, stderr, style } = options;
+    if (!isCacheableCommand(config.checkCache, command)) {
+        return undefined;
+    }
+    if (passthrough.includes('--no-cache')) {
+        stderr.write(`${formatMisplacedNoCacheWarning(command, style)}\n`);
+    }
+    if (passthrough.length > 0) {
+        writeDebugNote(`gate disabled: ${command} was passed arguments`, env, stderr);
+        return undefined;
+    }
+    const snapshot = resolveTreeSnapshot({ monorepoRoot, env });
+    if (!snapshot.ok) {
+        writeDebugNote(`gate disabled: ${snapshot.reason}`, env, stderr);
+        return undefined;
+    }
+    return snapshot.snapshot;
+}
+export function resolveCacheKey(options) {
+    const { env, snapshot, stderr, substitution } = options;
+    if (snapshot === undefined) {
+        return undefined;
+    }
+    if (substitution !== undefined) {
+        writeDebugNote(`gate disabled: devBin substituted \`${substitution}\``, env, stderr);
+        return undefined;
+    }
+    const result = computeCacheKey({
+        anchorDir: options.anchorDir,
+        command: options.command,
+        commandString: options.commandString,
+        env,
+        monorepoRoot: options.monorepoRoot,
+        nmrVersion: VERSION,
+        snapshot,
+    });
+    if (!result.ok) {
+        writeDebugNote(`gate disabled: ${result.reason}`, env, stderr);
+        return undefined;
+    }
+    return result.key;
+}
+export async function runGated(plan) {
+    const { commandString, key } = plan;
+    const { anchorDir, command, context, env, stderr, stdout } = plan.invocation;
+    const { config, monorepoRoot } = context;
+    let gate = await readGate(plan);
+    if (gate !== undefined && !plan.shouldBypassCache) {
+        const recalled = await recallPass(plan, gate);
+        if (recalled !== undefined) {
+            return recalled;
+        }
+    }
+    const hostResult = await runOnHost(plan, gate);
+    if (hostResult !== undefined) {
+        return hostResult;
+    }
+    const lock = await takeMachineLock(plan.invocation);
+    try {
+        if (lock?.waited === true && key !== undefined) {
+            gate = await readGate(plan);
+            if (gate !== undefined && !plan.shouldBypassCache) {
+                const recalled = await recallPass(plan, gate);
+                if (recalled !== undefined) {
+                    return recalled;
+                }
+            }
+        }
+        if (plan.overrideNotice !== undefined) {
+            stdout.write(plan.overrideNotice);
+        }
+        const runOptions = { ...plan.runOptions, env: addLockEnv(plan.runOptions.env ?? env, lock) };
+        const startedAtMs = Date.now();
+        const { exitCode, retainedOutput } = await runSteps(plan.steps, anchorDir, runOptions);
+        const durationMs = Date.now() - startedAtMs;
+        if (exitCode === 0 && gate !== undefined) {
+            await recordPass({
+                anchorDir,
+                buildOutputBefore: gate.buildOutputBefore,
+                command,
+                commandString,
+                config,
+                durationMs,
+                env,
+                key: gate.key,
+                monorepoRoot,
+                ownSteps: plan.ownSteps,
+                retainedOutput,
+                retentionKey: gate.retentionKey,
+                runId: plan.runId,
+                snapshot: gate.snapshot,
+                stderr,
+            });
+        }
+        return {
+            exitCode,
+            outcome: exitCode === 0 ? { outcome: 'passed', durationMs } : { outcome: 'failed', durationMs, exitCode },
+        };
+    }
+    finally {
+        await lock?.release();
+    }
+}
+function composeRecall(entry, retentionKey) {
+    return {
+        ageMs: Math.max(0, Date.now() - Date.parse(entry.recordedAt)),
+        entry,
+        savedMs: entry.durationMs,
+        ...(entry.retention?.key === retentionKey && { replay: entry.retention.replay }),
+    };
+}
+async function lookUpRecordedPass(options) {
+    const { anchorDir, buildOutput, command, env, key, monorepoRoot, stderr, targetHosts } = options;
+    const entry = await readCheckCacheEntry({ anchorDir, command, monorepoRoot });
+    if (entry === undefined) {
+        writeDebugNote(`running ${command}: no pass recorded for this scope`, env, stderr);
+        return undefined;
+    }
+    if (entry.key !== key) {
+        writeDebugNote(`running ${command}: the tree or its inputs changed since the last pass`, env, stderr);
+        return undefined;
+    }
+    if (entry.host !== undefined) {
+        if (targetHosts?.includes(entry.host) !== true) {
+            writeDebugNote(`running ${command}: the last pass was earned on ${entry.host}`, env, stderr);
+            return undefined;
+        }
+        return composeRecall(entry, options.retentionKey);
+    }
+    const [missingPackage] = buildOutput.missingPackages;
+    if (missingPackage !== undefined) {
+        writeDebugNote(`running ${command}: ${missingPackage} does not have build output`, env, stderr);
+        return undefined;
+    }
+    const stalePackage = findStaleBuildOutput(entry.buildDigests, buildOutput.digests);
+    if (stalePackage !== undefined) {
+        writeDebugNote(`running ${command}: ${stalePackage}'s build output came from a different tree`, env, stderr);
+        return undefined;
+    }
+    return composeRecall(entry, options.retentionKey);
+}
+async function readGate(plan) {
+    const { key, snapshot } = plan;
+    const { context, env, stderr, stdout } = plan.invocation;
+    if (key === undefined || snapshot === undefined) {
+        return undefined;
+    }
+    const quiet = plan.runOptions.quiet === true;
+    const channels = { stderr: resolveChannel(stderr, quiet), stdout: resolveChannel(stdout, quiet) };
+    return {
+        buildOutputBefore: await readBuildOutputState(context.monorepoRoot, context.config),
+        channels,
+        key,
+        retentionKey: computeRetentionKey({ channels, env, passKey: key }),
+        snapshot,
+    };
+}
+async function recallPass(plan, gate) {
+    const { anchorDir, command, context, env, stderr } = plan.invocation;
+    const { monorepoRoot } = context;
+    const recalledPass = await lookUpRecordedPass({
+        anchorDir,
+        buildOutput: gate.buildOutputBefore,
+        command,
+        env,
+        key: gate.key,
+        monorepoRoot,
+        retentionKey: gate.retentionKey,
+        stderr,
+        targetHosts: plan.target.kind === 'remote' ? plan.target.hosts : undefined,
+    });
+    if (recalledPass === undefined) {
+        return undefined;
+    }
+    const { entry, ...recall } = recalledPass;
+    if (recall.replay !== undefined) {
+        await certifyRetention({ anchorDir, command, entry, env, monorepoRoot, runId: plan.runId, stderr });
+    }
+    if (entry.host === undefined) {
+        await writePassManifestEntry({
+            anchorDir,
+            command,
+            commandString: plan.commandString,
+            durationMs: entry.durationMs,
+            env,
+            monorepoRoot,
+            replay: recall.replay,
+            stderr,
+            treeHash: gate.snapshot.treeId,
+        });
+    }
+    return {
+        exitCode: 0,
+        ...(entry.host !== undefined && { host: entry.host }),
+        outcome: { outcome: 'recalled', ...recall },
+    };
+}
+async function runOnHost(plan, gate) {
+    if (plan.target.kind !== 'remote') {
+        return undefined;
+    }
+    const hostRun = await runPlanOnHost(plan, plan.target);
+    if (hostRun.kind === 'here') {
+        return undefined;
+    }
+    if (hostRun.kind === 'refused') {
+        return { exitCode: hostRun.exitCode };
+    }
+    if (hostRun.exitCode === 0 && gate !== undefined) {
+        await recordDelegatedPass(plan, gate, hostRun);
+    }
+    return { exitCode: hostRun.exitCode, host: hostRun.host, outcome: hostRun.outcome };
+}

package/dist/esm/check-cache-key.d.ts

@@ -0,0 +1,47 @@
+import type { TreeSnapshot } from './check-cache.js';
+import type { OutputChannels } from './output-channel.js';
+/** The parts of the running interpreter that can change what a check concludes. */
+export interface RuntimeIdentity {
+    arch: string;
+    nodeVersion: string;
+    platform: string;
+}
+/** The interpreter on which this process is running, folded into every key and recorded alongside every pass. */
+export declare const CURRENT_RUNTIME: RuntimeIdentity;
+/**
+ * Folds everything that can change what a command concludes into one key: the tree's content, the command
+ * string that would run, the scope in which it would run, nmr's own version, the interpreter, what is installed,
+ * and the environment variables that a check can read. A hit is this key matching a recorded one, so an ingredient left
+ * out here is an ingredient that could change while the cache still claims a pass.
+ *
+ * Reports a reason instead of a key when the install fingerprint cannot be read, which disables the gate.
+ */
+export declare function computeCacheKey(options: {
+    anchorDir: string;
+    command: string;
+    commandString: string;
+    env: NodeJS.ProcessEnv;
+    monorepoRoot: string;
+    nmrVersion: string;
+    runtime?: RuntimeIdentity;
+    snapshot: TreeSnapshot;
+}): {
+    ok: true;
+    key: string;
+} | {
+    ok: false;
+    reason: string;
+};
+/**
+ * Folds what changes a transcript without changing a conclusion onto the pass key: the channel on which each of
+ * the command's output streams ran, and the environment variables through which a tool presents itself.
+ *
+ * The channel kind keeps a run at a terminal from replaying a piped recording. It is the channel's kind and not
+ * whether a terminal is attached: under quiet mode the child writes to pipes at a terminal, and its transcript
+ * is reproducible.
+ */
+export declare function computeRetentionKey(options: {
+    channels: OutputChannels;
+    env: NodeJS.ProcessEnv;
+    passKey: string;
+}): string;

package/dist/esm/check-cache-key.js

@@ -0,0 +1,77 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
+import process from 'node:process';
+export const CURRENT_RUNTIME = {
+    arch: process.arch,
+    nodeVersion: process.version,
+    platform: process.platform,
+};
+const INSTALL_FINGERPRINT_FILES = [
+    path.join('node_modules', '.modules.yaml'),
+    path.join('node_modules', '.pnpm', 'lock.yaml'),
+];
+const KEY_FORMAT = 'nmr-check-cache-v2';
+const KEYED_ENV_VARS = ['LANG', 'LC_ALL', 'NODE_OPTIONS', 'TZ'];
+const RETENTION_KEY_FORMAT = 'nmr-retention-v1';
+const RETENTION_KEYED_ENV_VARS = ['CI', 'COLUMNS', 'FORCE_COLOR', 'NO_COLOR', 'TERM'];
+export function computeCacheKey(options) {
+    const fingerprint = resolveInstallFingerprint(options.monorepoRoot);
+    if (!fingerprint.ok) {
+        return fingerprint;
+    }
+    const runtime = options.runtime ?? CURRENT_RUNTIME;
+    const scope = path.relative(options.monorepoRoot, options.anchorDir);
+    const parts = [
+        KEY_FORMAT,
+        options.snapshot.treeId,
+        scope === '' ? '.' : scope,
+        options.command,
+        options.commandString,
+        options.nmrVersion,
+        runtime.nodeVersion,
+        runtime.platform,
+        runtime.arch,
+        fingerprint.fingerprint,
+    ];
+    return { ok: true, key: digestParts([...parts, ...composeEnvParts(KEYED_ENV_VARS, options.env)]) };
+}
+export function computeRetentionKey(options) {
+    const parts = [
+        RETENTION_KEY_FORMAT,
+        options.passKey,
+        describeChannel(options.channels.stdout),
+        describeChannel(options.channels.stderr),
+    ];
+    return digestParts([...parts, ...composeEnvParts(RETENTION_KEYED_ENV_VARS, options.env)]);
+}
+function composeEnvParts(names, env) {
+    return names.flatMap((name) => {
+        const value = env[name];
+        return [name, value === undefined ? 'unset' : 'set', value ?? ''];
+    });
+}
+function describeChannel(channel) {
+    return channel === 'pipe' ? 'pipe' : 'descriptor';
+}
+function digestParts(parts) {
+    const hash = createHash('sha256');
+    for (const part of parts) {
+        hash.update(part);
+        hash.update('\0');
+    }
+    return hash.digest('hex');
+}
+function resolveInstallFingerprint(monorepoRoot) {
+    const hash = createHash('sha256');
+    for (const relativePath of INSTALL_FINGERPRINT_FILES) {
+        try {
+            hash.update(readFileSync(path.join(monorepoRoot, relativePath)));
+        }
+        catch {
+            return { ok: false, reason: `no install fingerprint: ${relativePath} is unreadable` };
+        }
+        hash.update('\0');
+    }
+    return { ok: true, fingerprint: hash.digest('hex') };
+}

package/dist/esm/check-cache-record.d.ts

@@ -0,0 +1,69 @@
+import type { Writable } from 'node:stream';
+import { type TreeSnapshot } from './check-cache.js';
+import { type BuildOutputState } from './check-cache-build-output.js';
+import { type EntryRef, type Retention } from './check-cache-entries.js';
+import { type RunCliResult, type RunPlan } from './invocation.js';
+import type { RetainedOutput } from './runner.js';
+import type { Step } from './steps.js';
+import type { NmrConfig } from './types.js';
+/**
+ * Records a pass that `host` earned, unless a pass earned here under the same key would still be recalled, because
+ * that pass satisfies every invocation that the host pass would. One whose build output is missing or came from
+ * another tree no longer satisfies any invocation, and the host pass replaces it.
+ *
+ * A host pass is recorded without build digests, because this machine's build output says nothing about the host's.
+ * The caller has already held the host's tree to the tree on which the key was computed.
+ */
+export declare function recordHostPass(options: {
+    commandString: string;
+    config: NmrConfig;
+    durationMs: number;
+    env: NodeJS.ProcessEnv;
+    host: string;
+    key: string;
+    monorepoRoot: string;
+    ref: EntryRef;
+    retainedOutput: RetainedOutput | undefined;
+    retention: Retention | undefined;
+    snapshot: TreeSnapshot;
+    stderr: Writable;
+}): Promise<void>;
+/**
+ * Records a pass, unless what the check was asked about moved while it ran: a rewritten file describes a tree
+ * that no longer exists, and build output that changed leaves it unclear over which output the pass was earned.
+ * Recording either would certify content against which nothing ran. The recorded key is the one that the
+ * snapshot produced, so every entry from one invocation refers to one tree.
+ *
+ * A cache that cannot be written is not worth failing a green run over: `recordPass` reports that failure in the
+ * debug note.
+ *
+ * The retention that a skip replays is composed here for the same reason and after the same tests: a pass that is
+ * not recorded must not leave behind an excerpt that claims it was. A composite's assembly is built here rather
+ * than when it skips, to ensure that every excerpt in it was certified during the run whose pass contains it.
+ */
+export declare function recordPass(options: {
+    anchorDir: string;
+    buildOutputBefore: BuildOutputState;
+    command: string;
+    commandString: string;
+    config: NmrConfig;
+    durationMs: number;
+    env: NodeJS.ProcessEnv;
+    key: string;
+    monorepoRoot: string;
+    retainedOutput: RetainedOutput | undefined;
+    ownSteps: readonly Step[];
+    retentionKey: string;
+    runId: string;
+    snapshot: TreeSnapshot;
+    stderr: Writable;
+}): Promise<void>;
+/**
+ * Prints what the current scope has recorded for one command, and reports what it exits with.
+ *
+ * A refusal ends the invocation non-zero, which is what tells a caller that nothing on stdout is the run that it
+ * asked for. Under a delegate it does not: a fan-out asks every selected scope, and a scope that never ran the
+ * command is a gap in a survey rather than a failure of one, so bailing there would hide every scope that has
+ * something to show.
+ */
+export declare function reportRecording(plan: RunPlan): Promise<RunCliResult>;