@hominis/fireforge 0.45.0 → 0.46.1

package/dist/src/core/file-lock-owner.js

@@ -0,0 +1,259 @@
+// SPDX-License-Identifier: EUPL-1.2
+/**
+ * Owner record of a `withFileLock` directory: the file that says which
+ * process holds the lock, so stale recovery can tell a dead holder from a
+ * slow one and release can refuse to remove a lock it no longer owns.
+ *
+ * Split out of `file-lock.ts` so the record's format, its write-and-verify
+ * contract, and the PID-reuse guard live in one place with their own tests.
+ */
+import { readFileSync } from 'node:fs';
+import { readFile, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { GeneralError } from '../errors/base.js';
+import { isProcessAlive, toError } from '../utils/errors.js';
+import { verbose } from '../utils/logger.js';
+import { sleep } from '../utils/sleep.js';
+/**
+ * Filename of a lock directory's owner record, relative to the lock dir.
+ *
+ * Re-exported by `file-lock.ts` so external readers (`tree-store.ts`) share
+ * the constant instead of re-spelling `'pid'`. The file format is
+ * documented on {@link readLockOwner}.
+ */
+export const LOCK_PID_FILE = 'pid';
+/**
+ * Prefix of the mechanical owner-record line that carries the acquisition
+ * wall time. It lives among the metadata lines (line 3 onward) so older
+ * readers, which treat everything past line 2 as free-form diagnostics,
+ * still parse the record. The current reader lifts it out of `metadata`.
+ * Diagnostic only. See {@link START_TICK_PREFIX} for the liveness field.
+ */
+const ACQUIRED_AT_PREFIX = 'acquired-at-ms=';
+/**
+ * Prefix of the owner-record line carrying the holder's own process start
+ * tick (`/proc/self/stat` field 22, in USER_HZ since boot). Written only
+ * where procfs exposes it (Linux). Boot-relative ticks are what the
+ * PID-reuse guard compares: unlike a wall-clock time they are immune to
+ * the clock being stepped by NTP/chrony/`date -s` between acquisition and
+ * probe, and a process's start tick never changes for its lifetime, so the
+ * comparison is exact rather than a tolerance window.
+ */
+const START_TICK_PREFIX = 'start-tick=';
+/**
+ * How many times the post-write read-back is retried when the read itself
+ * errors (not when the record is absent or differs). An on-access scanner
+ * on Windows briefly holding the just-written file (`EBUSY`), or a FUSE
+ * mount's momentary `EIO`, must not be indistinguishable from "the record
+ * never landed": that verdict releases a lock this process has correctly
+ * acquired and aborts the run before it starts.
+ */
+const READ_BACK_ATTEMPTS = 5;
+/** Pause between read-back attempts. */
+const READ_BACK_RETRY_MS = 20;
+/**
+ * Parses the on-disk record text. Returns `undefined` when the PID line
+ * does not parse as a finite integer.
+ */
+function parseLockOwnerRecord(pidContent) {
+    const [pidLine, tokenLine, ...trailingLines] = pidContent.split('\n');
+    const pid = parseInt((pidLine ?? '').trim(), 10);
+    if (!Number.isFinite(pid))
+        return undefined;
+    const metadata = [];
+    let acquiredAtMs;
+    let startTick;
+    for (const rawLine of trailingLines) {
+        const line = rawLine.trim();
+        if (line.length === 0)
+            continue;
+        if (line.startsWith(ACQUIRED_AT_PREFIX)) {
+            const parsed = Number.parseInt(line.slice(ACQUIRED_AT_PREFIX.length), 10);
+            if (Number.isFinite(parsed))
+                acquiredAtMs ??= parsed;
+            continue;
+        }
+        if (line.startsWith(START_TICK_PREFIX)) {
+            const parsed = Number.parseInt(line.slice(START_TICK_PREFIX.length), 10);
+            if (Number.isFinite(parsed))
+                startTick ??= parsed;
+            continue;
+        }
+        metadata.push(line);
+    }
+    const token = tokenLine?.trim();
+    const owner = { present: true, pid, metadata };
+    if (token !== undefined && token.length > 0)
+        owner.token = token;
+    if (acquiredAtMs !== undefined)
+        owner.acquiredAtMs = acquiredAtMs;
+    if (startTick !== undefined)
+        owner.startTick = startTick;
+    return owner;
+}
+/**
+ * Reads the owner PID (line 1), acquisition token (line 2), the mechanical
+ * `acquired-at-ms=` / `start-tick=` lines (when present), and diagnostic
+ * metadata (the remaining lines 3+) from a lock directory's PID file.
+ * Returns `{ present: false }` when the PID file is missing, unreadable, or
+ * the PID does not parse as a finite integer (caller falls back to the
+ * age-only staleness heuristic).
+ *
+ * File format note: external readers do `parseInt(content.trim(), 10)`,
+ * which parses the leading digits of a multi-line file, so the token and
+ * mechanical lines stay compatible.
+ *
+ * @param lockPath - Lock directory whose owner record to read
+ * @returns The parsed owner record, or `{ present: false }`
+ */
+export async function readLockOwner(lockPath) {
+    try {
+        const owner = parseLockOwnerRecord(await readFile(join(lockPath, LOCK_PID_FILE), 'utf-8'));
+        if (owner !== undefined)
+            return owner;
+    }
+    catch {
+        // PID file missing or unreadable. Treat as absent.
+    }
+    return { present: false };
+}
+/**
+ * Reads the just-written record back, retrying a read error a few times
+ * before giving up. A missing or malformed record is returned as-is (the
+ * caller treats it as a failed write). Only errno-level failures retry.
+ */
+async function readBackLockOwner(lockPath) {
+    let lastError;
+    for (let attempt = 0; attempt < READ_BACK_ATTEMPTS; attempt++) {
+        try {
+            return parseLockOwnerRecord(await readFile(join(lockPath, LOCK_PID_FILE), 'utf-8'));
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt < READ_BACK_ATTEMPTS - 1)
+                await sleep(READ_BACK_RETRY_MS);
+        }
+    }
+    throw new Error(`owner record could not be read back: ${toError(lastError).message}`);
+}
+/**
+ * Stamps this process's ownership (PID, per-acquisition token, acquisition
+ * time, own start tick, diagnostic metadata) into a lock directory it has
+ * just created, and reads the record back to prove it landed.
+ *
+ * The write is fatal. It used to be best-effort, on the theory that stale
+ * recovery would fall back to the age heuristic, but that heuristic reaps
+ * any owner-less lock after five minutes, and the build lock legitimately
+ * holds for hours. A holder whose record failed to write was therefore
+ * reaped mid-build and a second process entered the critical section. On
+ * failure the just-created lock directory is removed (nothing else can own
+ * it yet) and a {@link GeneralError} names the cause.
+ *
+ * @param lockPath - Lock directory this process has just `mkdir`'d
+ * @param token - Per-acquisition UUID, later checked by release
+ * @param ownerMetadata - Extra human-diagnostic lines (see `FileLockOptions.ownerMetadata`)
+ * @throws {GeneralError} When the record cannot be written or does not read back as written
+ */
+export async function writeLockOwner(lockPath, token, ownerMetadata) {
+    const metadata = ownerMetadata
+        ?.map((line) => line.replace(/\r?\n/g, ' ').trim())
+        .filter((line) => line.length > 0 &&
+        !line.startsWith(ACQUIRED_AT_PREFIX) &&
+        !line.startsWith(START_TICK_PREFIX)) ?? [];
+    const ownStartTick = readProcessStartTick(process.pid);
+    const mechanicalLines = [
+        `${ACQUIRED_AT_PREFIX}${Date.now()}`,
+        ...(ownStartTick === undefined ? [] : [`${START_TICK_PREFIX}${ownStartTick}`]),
+    ];
+    const record = `${process.pid}\n${token}\n${[...mechanicalLines, ...metadata].map((line) => `${line}\n`).join('')}`;
+    let failure;
+    try {
+        await writeFile(join(lockPath, LOCK_PID_FILE), record, 'utf-8');
+        const readBack = await readBackLockOwner(lockPath);
+        if (readBack !== undefined && readBack.pid === process.pid && readBack.token === token) {
+            return;
+        }
+        failure = new Error('owner record did not read back as written');
+    }
+    catch (error) {
+        failure = error;
+    }
+    try {
+        await rm(lockPath, { recursive: true, force: true });
+    }
+    catch (cleanupError) {
+        verbose(`Could not remove unowned lock ${lockPath}: ${toError(cleanupError).message}`);
+    }
+    throw new GeneralError(`Could not record ownership of lock ${lockPath}: ${toError(failure).message}. ` +
+        'The lock was released without running the operation — a lock with no readable owner ' +
+        'would be reaped as stale after five minutes and a second process could enter the ' +
+        'critical section. Check permissions and free space on the lock directory, then retry.', failure);
+}
+/**
+ * Boot-relative start tick of a process, where the platform exposes it
+ * cheaply.
+ *
+ * Linux only: field 22 of `/proc/<pid>/stat`, in USER_HZ since boot.
+ * Elsewhere (macOS, Windows) obtaining a start time means spawning
+ * `ps`/`wmic` per probe, which is not worth paying on every poll of a
+ * contended lock. Those platforms return `undefined` and liveness stays
+ * PID-only.
+ *
+ * Not converted to wall-clock time via `/proc/stat` `btime`, on purpose:
+ * on current kernels `btime` is derived from the realtime clock and moves
+ * with every clock step, so a wall-clock comparison against a recorded
+ * acquisition time misjudges a live holder as a reused PID after NTP steps
+ * the clock forward, and reaps a lock that is in use.
+ *
+ * @param pid - Process to look up
+ * @param procRoot - procfs mount, overridable for tests
+ * @returns Start tick, or `undefined` when unobtainable
+ */
+export function readProcessStartTick(pid, procRoot = '/proc') {
+    try {
+        const stat = readFileSync(join(procRoot, String(pid), 'stat'), 'utf-8');
+        // `comm` (field 2) is parenthesised and may itself contain spaces or
+        // parentheses, so split only what follows the last closing paren:
+        // field 3 (state) is then index 0 and field 22 (starttime) index 19.
+        const startTick = Number.parseInt(stat
+            .slice(stat.lastIndexOf(')') + 1)
+            .trim()
+            .split(/\s+/)[19] ?? '', 10);
+        return Number.isFinite(startTick) ? startTick : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Whether the process named by a lock's owner record is the same process
+ * that acquired the lock, not merely a live process wearing its PID.
+ *
+ * `isProcessAlive` answers "does this PID exist". After a holder crashes,
+ * the kernel may hand its PID to a long-lived daemon, and a PID-only check
+ * then honours the dead lock until the waiter's timeout. Where the record
+ * carries the writer's start tick and the candidate's own tick is cheaply
+ * obtainable (see {@link readProcessStartTick}), the two must be equal: a
+ * different tick is a different process. Records without a start tick, and
+ * platforms without one, keep the PID-only answer.
+ *
+ * @param owner - Owner record fields that identify the holder
+ * @param procRoot - procfs mount, overridable for tests
+ * @returns True when the original holder is still running (or cannot be proven gone)
+ */
+export function isLockOwnerAlive(owner, procRoot) {
+    if (!isProcessAlive(owner.pid))
+        return false;
+    if (owner.startTick === undefined)
+        return true;
+    const currentStartTick = readProcessStartTick(owner.pid, procRoot);
+    if (currentStartTick === undefined)
+        return true;
+    if (currentStartTick !== owner.startTick) {
+        verbose(`Lock owner PID ${owner.pid} started at tick ${currentStartTick}, but the ` +
+            `record was written by a process started at tick ${owner.startTick} — the PID was ` +
+            'reused; treating the holder as dead');
+        return false;
+    }
+    return true;
+}

package/dist/src/core/file-lock.d.ts

@@ -1,3 +1,5 @@
+import { LOCK_PID_FILE } from './file-lock-owner.js';
+export { LOCK_PID_FILE };
 /**
  * Snapshot of the current lock holder handed to {@link FileLockOptions.onWaitProgress}.
  * `metadata` carries the pid file's human-diagnostic lines (line 3 onward,
@@ -12,15 +14,15 @@ export interface LockHolder {
 /**
  * Live queue state around a contended lock.
  *
- * The lock itself behaves correctly under several concurrent sessions; the
- * gap this closes is VISIBILITY. Waiters advertise themselves in a sibling
+ * The lock itself behaves correctly under several concurrent sessions. The
+ * gap this closes is visibility. Waiters advertise themselves in a sibling
  * directory so both the wait output and `fireforge status --lock` can say
  * how many are queued and how many are ahead of you.
  */
 export interface LockQueueState {
     /** Live waiters currently queued for the lock, including this process. */
     depth: number;
-    /** Live waiters that started waiting BEFORE this process. */
+    /** Live waiters that started waiting before this process. */
     ahead: number;
 }
 export interface FileLockOptions {
@@ -29,7 +31,7 @@ export interface FileLockOptions {
     /**
      * When set, the poll interval doubles after every contended poll (starting
      * from `pollMs`) up to this cap. When absent, polling stays at the fixed
-     * `pollMs` interval — zero behavior change for existing callers.
+     * `pollMs` interval, with zero behavior change for existing callers.
      */
     pollMaxMs?: number;
     staleMs?: number;
@@ -58,17 +60,17 @@ export interface FileLockOptions {
         queue?: LockQueueState | undefined;
     }) => void;
     /**
-     * When set, a wait whose QUEUE POSITION is still improving is granted a
+     * When set, a wait whose queue position is still improving is granted a
      * fresh `timeoutMs` budget on each advance, up to `maxWaitMs` total.
      *
      * A fixed wall-clock budget against a queue whose depth the tool knows is
      * the wrong shape under several concurrent sessions: a wait that expires
      * one position from the head pays the entire wait and gets nothing. The
-     * distinction that matters is not "how long have I waited" but "is this
-     * queue moving" — a stalled queue still starves on the original budget,
+     * question that matters is whether the queue is moving, not how long the
+     * wait has been. A stalled queue still starves on the original budget,
      * which is the case the timeout exists for.
      *
-     * Requires the waiter registry (see {@link LockQueueState}); a lock with
+     * Requires the waiter registry (see {@link LockQueueState}). A lock with
      * no readable queue never extends, because nothing observed an advance.
      */
     extendWhileAdvancing?: {
@@ -80,7 +82,7 @@ export interface FileLockOptions {
      * Separate from {@link onWaitProgress} on purpose: an extension is the one
      * moment the budget the operator asked for stops being the budget in
      * force, and a wait that silently outlives its stated timeout is
-     * indistinguishable from a broken one. `budgetMs` is the new TOTAL budget
+     * indistinguishable from a broken one. `budgetMs` is the new total budget
      * measured from the start of the wait, so it can be compared directly
      * against the requested figure.
      */
@@ -90,18 +92,11 @@ export interface FileLockOptions {
     }) => void;
     onTimeoutMessage?: string;
     onStaleLockMessage?: (ageMs: number) => string | undefined;
-    /** Extra owner-file lines for human diagnostics; ignored by lock mechanics. */
+    /** Extra owner-file lines for human diagnostics. Lock mechanics ignore them. */
     ownerMetadata?: readonly string[];
 }
 /** Derives the sibling lock-directory path used to guard a file-based resource. */
 export declare function createSiblingLockPath(filePath: string, suffix?: string): string;
-/**
- * Filename of a lock directory's owner record, relative to the lock dir.
- *
- * Exported so external readers (`tree-store.ts`) share the constant instead of
- * re-spelling `'pid'`; the file *format* is documented on {@link readLockOwner}.
- */
-export declare const LOCK_PID_FILE = "pid";
 /**
  * Reads the live waiter queue around `lockPath`, dropping entries whose
  * process is gone (a killed waiter must not inflate the depth forever).
@@ -133,11 +128,11 @@ export declare function readLockStatus(lockPath: string): Promise<LockStatusSnap
  * Removes a lock directory only if this process still owns it, for callers
  * that hold no acquisition token.
  *
- * The ownership half of {@link releaseLock}, minus the token check — a
+ * The ownership half of {@link releaseLock}, minus the token check: a
  * signal-time sweeper knows which lock paths its own operations opened, but
  * not the per-acquisition UUID those acquisitions minted. The PID check is
  * still the one that matters: it stops the sweeper deleting a lock a
- * DIFFERENT process acquired in the window between our operation dying and
+ * different process acquired in the window between our operation dying and
  * the sweep running.
  *
  * Never throws: callers are shutdown paths where a slow or failing I/O must