@parcha/pi-durable-disk 0.1.0-beta.15 → 0.1.0-beta.17

package/dist/place.js

@@ -2,11 +2,12 @@
 // kill (agentrun's reference demo) is the same for every place; the place decides how. `localPlace` runs the instance on
 // a directory of this machine (`run --dir`, no disk and no key); `archilPlace` runs it on an Archil disk through a host
 // driver, mints each attempt's mount token itself and takes the run over after a kill. The types match the reference's
-// place seam (examples/reference/place.ts) structurally. The supervisor reads nothing under the run's directory: the
+// place seam (@parcha/agentrun-reference/place) structurally. The supervisor reads nothing under the run's directory: the
 // outcome, the run's record and its delegations come through the place.
 import { spawn } from "node:child_process";
-import { existsSync, readFileSync } from "node:fs";
-import { extname, join, posix, resolve } from "node:path";
+import { createHash, randomUUID } from "node:crypto";
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeSync } from "node:fs";
+import { dirname, extname, join, posix, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { DEFAULT_MOUNT_ROOT, findDelegations, mintMountToken, pathlessResolver, removeMountToken, runPath } from "./claim.js";
 import { reapCommands } from "./commands.js";
@@ -48,6 +49,7 @@ function outcomeOf(record) {
         return undefined;
     const detail = record.detail !== null && typeof record.detail === "object" && !Array.isArray(record.detail) ? record.detail : {};
     const delivered = record.status === "done" && "record" in detail && detail.record !== null && detail.record !== undefined;
+    const models = modelsOf(detail.models);
     return {
         status: delivered ? "complete" : "failed",
         generation: record.generation,
@@ -55,8 +57,22 @@ function outcomeOf(record) {
         ...(typeof detail.message === "string" ? { message: detail.message } : {}),
         ...("record" in detail ? { record: detail.record } : {}),
         ...(typeof detail.storeSha256 === "string" ? { storeSha256: detail.storeSha256 } : {}),
+        ...(models ? { models } : {}),
     };
 }
+/**
+ * The role-to-model ids an outcome carries: a copy of `value` when it is a plain object whose values are all strings,
+ * else undefined, so a malformed report is left out and never stands in as `{}`.
+ */
+function modelsOf(value) {
+    if (value === null || typeof value !== "object" || Array.isArray(value))
+        return undefined;
+    const proto = Object.getPrototypeOf(value);
+    if (proto !== Object.prototype && proto !== null)
+        return undefined;
+    const entries = Object.entries(value);
+    return entries.every(([, id]) => typeof id === "string") ? Object.fromEntries(entries) : undefined;
+}
 const pathsUnder = (id, root) => ({ id, root, work: join(root, "work"), store: join(root, "store") });
 /**
  * The run on a directory of this machine: each runner is `pi-durable-disk run --dir <root> --id <id> --app <app>`, a
@@ -281,13 +297,20 @@ function requestIn(text) {
         return undefined;
     }
 }
-/** The `outcome` of a `{"outcome":{...}}` line with at least a status; undefined for any other line. */
+/**
+ * The `outcome` of a `{"outcome":{...}}` line with at least a status; undefined for any other line. Its `models` stays
+ * only when it is a plain object of strings (`modelsOf`), as `run.json`'s does.
+ */
 function outcomeLine(line) {
     if (!line.startsWith("{"))
         return undefined;
     try {
         const parsed = JSON.parse(line);
-        return parsed.outcome && typeof parsed.outcome === "object" && typeof parsed.outcome.status === "string" ? parsed.outcome : undefined;
+        if (!parsed.outcome || typeof parsed.outcome !== "object" || typeof parsed.outcome.status !== "string")
+            return undefined;
+        const { models: reported, ...outcome } = parsed.outcome;
+        const models = modelsOf(reported);
+        return models ? { ...outcome, models } : outcome;
     }
     catch {
         return undefined;
@@ -295,11 +318,263 @@ function outcomeLine(line) {
 }
 /** Where in the run's directory an Archil place's runners write their kill requests. */
 const REQUESTS = "signals";
-const requestName = (attempt) => {
+/** Where in the run's directory an Archil place's runners record, one directory per attempt. */
+const RECORDS = "record";
+const BLOB_NAME = /^([0-9a-f]{64})\.[A-Za-z0-9]{1,16}$/;
+const SPOOL_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,200}\.jsonl$/;
+const NEWLINE = 0x0a;
+/** A message for a recording's line: at most 240 characters, so it never carries more than the line can hold. */
+const said = (text) => text.slice(0, MESSAGE_KEEP);
+/**
+ * Every byte of `bytes` to `fd`: a write may take fewer bytes than it was given, so it goes on from where the last one
+ * stopped. A write that takes none fails, as one that throws (a full disk) does.
+ */
+export function writeAll(fd, bytes, write = writeSync) {
+    for (let at = 0; at < bytes.length;) {
+        const n = write(fd, bytes, at, bytes.length - at);
+        if (!(n > 0))
+            throw new PlaceError("CARRY_FAILED", `a write took no bytes, ${bytes.length - at} of ${bytes.length} left`);
+        at += n;
+    }
+}
+/** `bytes` into `file` whole: a temporary file, fsync, rename, then the directory's fsync. */
+function writeWhole(file, bytes) {
+    const dir = dirname(file);
+    mkdirSync(dir, { recursive: true });
+    const tmp = join(dir, `.${randomUUID()}.tmp`);
+    const fd = openSync(tmp, "wx", 0o644);
+    try {
+        writeAll(fd, bytes);
+        fsyncSync(fd);
+    }
+    catch (error) {
+        closeSync(fd);
+        rmSync(tmp, { force: true });
+        throw error;
+    }
+    closeSync(fd);
+    renameSync(tmp, file);
+    syncDir(dir);
+}
+/** `bytes` at the end of `file`, fsynced; a file it creates is made durable in its directory too. */
+function appendDurably(file, bytes) {
+    const dir = dirname(file);
+    const created = !existsSync(file);
+    mkdirSync(dir, { recursive: true });
+    const fd = openSync(file, "a", 0o644);
+    try {
+        writeAll(fd, bytes);
+        fsyncSync(fd);
+    }
+    finally {
+        closeSync(fd);
+    }
+    if (created)
+        syncDir(dir);
+}
+/** How long a carrier waits for another one appending to the same destination file. */
+const CARRY_LOCK_WAIT_MS = 5_000;
+/** A lock file with no holder written yet (its taker is between the create and the write) counts as held this long. */
+const CARRY_LOCK_UNNAMED_MS = 10_000;
+const alive = (pid) => {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === "EPERM";
+    }
+};
+/**
+ * `fn` while holding `lock`, a file created exclusively that names its holder's pid, so two carriers on one destination
+ * (two places, two processes) never append at once. A lock whose holder is gone (its process ended) is taken over; one a
+ * live holder keeps past CARRY_LOCK_WAIT_MS is CARRY_FAILED. `fn` is synchronous, so the lock is held only for it.
+ */
+async function withCarryLock(lock, fn) {
+    for (const t0 = Date.now();;) {
+        let fd;
+        try {
+            fd = openSync(lock, "wx", 0o644);
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+        }
+        if (fd !== undefined) {
+            // Once created, the lock goes on every way out: a failed write of the pid or close of the file included.
+            try {
+                try {
+                    writeAll(fd, new TextEncoder().encode(String(process.pid)));
+                }
+                finally {
+                    closeSync(fd);
+                }
+                fn();
+            }
+            finally {
+                rmSync(lock, { force: true });
+            }
+            return;
+        }
+        let holder = 0;
+        let ageMs = 0;
+        try {
+            holder = Number(readFileSync(lock, "utf8").trim());
+            ageMs = Date.now() - statSync(lock).mtimeMs;
+        }
+        catch (error) {
+            // Released meanwhile: take it on the next turn. Any other failure to read it (its mode, the disk) is the carry's.
+            if (error.code === "ENOENT" && Date.now() - t0 <= CARRY_LOCK_WAIT_MS)
+                continue;
+            throw new PlaceError("CARRY_FAILED", said(`reading the carry lock ${lock} failed: ${error?.message ?? error}`), { cause: error });
+        }
+        const gone = Number.isSafeInteger(holder) && holder > 0 ? !alive(holder) : ageMs > CARRY_LOCK_UNNAMED_MS;
+        if (gone) {
+            rmSync(lock, { force: true });
+            continue;
+        }
+        if (Date.now() - t0 > CARRY_LOCK_WAIT_MS)
+            throw new PlaceError("CARRY_FAILED", said(`another carrier (pid ${holder || "unnamed"}) holds ${lock} past ${CARRY_LOCK_WAIT_MS} ms`));
+        await new Promise((resolve) => setTimeout(resolve, 20));
+    }
+}
+function syncDir(dir) {
+    const fd = openSync(dir, "r");
+    try {
+        fsyncSync(fd);
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+/** The blob hashes (SHA-256, hex) a recording's line names in its `blobs`; none for a line that is not JSON or names none. */
+function blobsOf(line) {
+    try {
+        const v = JSON.parse(new TextDecoder().decode(line));
+        return Array.isArray(v?.blobs) ? v.blobs.filter((b) => typeof b === "string") : [];
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Brings home into `into` what the runners recorded under `prefix` (`runs/<id>/record/`), read through the control (S3
+ * ListObjects and GetObject: nothing takes the run's claim). It keeps no state: a blob is home when `into/blobs/<name>`
+ * exists, and a spool file's carried bytes are what `into/.spool/<name>` holds, so a second call, or a call from another
+ * process, carries nothing twice. Carriers that overlap on one destination never append at once: the place runs one call
+ * at a time per destination, and every append happens under a lock file beside its spool file (`.<name>.lock`, naming
+ * its holder's pid), where the destination's size is read again so only the lines past it are appended. A lock whose
+ * holder is gone is taken over; one held past 5 s is CARRY_FAILED. Every write is carried to its last byte.
+ * - Blobs first: each one not yet home is fetched, checked against the hash its name carries, and written whole into
+ *   `into/blobs`. One whose bytes do not match is skipped; everything else is brought home, then the call throws
+ *   BLOB_MISMATCH, and the next call tries that blob again.
+ * - Then lines: the new whole lines of each `.spool/<name>.jsonl`, the recorder's segments (`<stream>.<n>.jsonl`), each
+ *   with its own offset: what `into/.spool/<name>` already holds. A file is fetched only when it grew since this place
+ *   last fetched it (the SDK has no ranged read). A line whose blobs are not home yet waits for the next call, so no line arrives before its blob; on
+ *   `final` it is carried anyway, and its missing blobs are CARRY_FAILED once everything else is home.
+ * - A listing or fetch that fails is CARRY_FAILED. The recording is on the run's disk, not in the box, so a lost box
+ *   costs only what it had not fsynced: this place never throws BOX_GONE.
+ */
+async function carryHome(control, prefix, into, final, fetched) {
+    const fetch = async (key) => {
+        try {
+            return await control.getObject(key);
+        }
+        catch (error) {
+            throw new PlaceError("CARRY_FAILED", said(`reading ${key.slice(prefix.length)} of the run's recording failed: ${error?.message ?? error}`), { cause: error });
+        }
+    };
+    let objects;
+    try {
+        objects = (await control.listObjects(prefix, { recursive: true })).objects;
+    }
+    catch (error) {
+        throw new PlaceError("CARRY_FAILED", said(`listing the run's recording failed: ${error?.message ?? error}`), { cause: error });
+    }
+    // `<attempt>/blobs/<name>` and `<attempt>/.spool/<name>.jsonl`; anything else (`started`, a box's temporary file) is not carried.
+    const parts = (key) => key.slice(prefix.length).split("/");
+    const blobs = objects.filter((o) => {
+        const p = parts(o.key);
+        return p.length === 3 && p[1] === "blobs" && BLOB_NAME.test(p[2]);
+    });
+    const spools = objects.filter((o) => {
+        const p = parts(o.key);
+        return p.length === 3 && p[1] === ".spool" && SPOOL_NAME.test(p[2]);
+    });
+    const home = (name) => existsSync(join(into, "blobs", name));
+    const mismatched = [];
+    for (const blob of blobs) {
+        const name = parts(blob.key)[2];
+        if (home(name))
+            continue;
+        const bytes = await fetch(blob.key);
+        if (createHash("sha256").update(bytes).digest("hex") !== BLOB_NAME.exec(name)[1]) {
+            mismatched.push(name);
+            continue;
+        }
+        writeWhole(join(into, "blobs", name), bytes);
+    }
+    // A line names its blobs by hash; the file is `<hash>.<ext>`.
+    const homeHashes = new Set(existsSync(join(into, "blobs")) ? readdirSync(join(into, "blobs")).map((n) => BLOB_NAME.exec(n)?.[1]).filter((h) => h !== undefined) : []);
+    const waiting = new Set();
+    for (const spool of spools) {
+        const name = parts(spool.key)[2];
+        const dest = join(into, ".spool", name);
+        const carried = existsSync(dest) ? statSync(dest).size : 0;
+        // Not fetched again while its size is what this place last fetched and nothing in it waited for a blob (a partial
+        // last line keeps it bigger than what is home). Never skipped on `final`. Only a saving: what is home is still what
+        // the destination holds.
+        // Per destination: what this place fetched for one `into` says nothing of another.
+        const cacheKey = `${resolve(into)}\0${spool.key}`;
+        if (spool.size !== undefined && (spool.size <= carried || (!final && fetched.get(cacheKey) === spool.size)))
+            continue;
+        const bytes = await fetch(spool.key);
+        if (bytes.length <= carried)
+            continue;
+        let heldForBlob = false;
+        let end = carried;
+        for (let at = carried; at < bytes.length;) {
+            const nl = bytes.indexOf(NEWLINE, at);
+            if (nl < 0)
+                break; // a line still being written
+            const missing = blobsOf(bytes.subarray(at, nl)).filter((hash) => !homeHashes.has(hash));
+            if (missing.length > 0 && !final) {
+                heldForBlob = true;
+                break;
+            }
+            for (const b of missing)
+                waiting.add(b);
+            end = nl + 1;
+            at = nl + 1;
+        }
+        if (end > carried) {
+            // Another carrier on this destination (another place, another process) may have appended since `carried` was read.
+            // Under the file's lock, what the destination holds now decides: only the lines past it are appended.
+            mkdirSync(dirname(dest), { recursive: true });
+            await withCarryLock(join(dirname(dest), `.${name}.lock`), () => {
+                const now = existsSync(dest) ? statSync(dest).size : 0;
+                if (end > now)
+                    appendDurably(dest, bytes.subarray(now, end));
+            });
+        }
+        if (heldForBlob)
+            fetched.delete(cacheKey);
+        else
+            fetched.set(cacheKey, bytes.length);
+    }
+    if (mismatched.length > 0) {
+        throw new PlaceError("BLOB_MISMATCH", said(`${mismatched.length} blob(s) of the run's recording do not hash to their names: ${mismatched.join(", ")}`));
+    }
+    if (final && waiting.size > 0) {
+        throw new PlaceError("CARRY_FAILED", said(`${waiting.size} blob(s) the run's recording names never reached its disk: ${[...waiting].join(", ")}`));
+    }
+}
+const countFrom1 = (attempt) => {
     if (!Number.isSafeInteger(attempt) || attempt < 1)
         throw new PlaceError("INVALID_ARGUMENT", `attempt ${attempt} is not a count from 1`);
-    return `kill-${attempt}.json`;
+    return attempt;
 };
+const requestName = (attempt) => `kill-${countFrom1(attempt)}.json`;
 /**
  * The run on an Archil disk, its instance run by a host driver. `token(attempt)` mints the attempt's mount token through
  * the place's control, and the runner it is handed to mounts with it. A start is `ensureRunning` (it creates the run's
@@ -330,8 +605,40 @@ export function archilPlace(options) {
         const paths = { id: ref.id, root, work: posix.join(root, "work"), store: posix.join(root, "store") };
         const minted = new Map();
         const handles = [];
+        /** Every handle the place started, released or not: the boxes it reports. */
+        const started = [];
         const killed = new Map();
         let resolver = pathlessResolver(control);
+        const records = `${runPath(ref.id)}/${RECORDS}/`;
+        const recording = typeof options.control.listObjects === "function";
+        const owner = options.ensure?.create;
+        /**
+         * Before attempt `gen`'s runner starts: the EMPTY directories it writes into, `signals/` and, with `recordHome`,
+         * `record/<gen>/.spool/` and `record/<gen>/blobs/`, made over the control (PutObject with the owner, as createRunDir
+         * does: an exec mkdir would make them root's) and owned by the run user (`ensure.create`), in ensureRunning's window
+         * where no delegation holds the run (a held run refuses them: DIRS_REFUSED). A directory a process
+         * made by path on an Archil mount was seen to refuse file creates afterwards (ENOENT), and one made over the control
+         * does not. Every file in them stays the runner's (its kill request, its started, its spool), and a runner that
+         * makes them itself finds them made. Without an owner nothing is made: a directory the run user does not own would
+         * refuse the runner's files.
+         */
+        const makeDirs = async (gen) => {
+            if (!owner)
+                return;
+            const run = `${runPath(ref.id)}/`;
+            const dirs = [`${run}${REQUESTS}/`, ...(recording ? [`${run}${RECORDS}/`, `${run}${RECORDS}/${gen}/`, `${run}${RECORDS}/${gen}/.spool/`, `${run}${RECORDS}/${gen}/blobs/`] : [])];
+            for (const dir of dirs) {
+                try {
+                    if (!(await control.headObject(dir)))
+                        await control.putObject(dir, "", { uid: owner.uid, gid: owner.gid, mode: owner.mode ?? 0o755 });
+                }
+                catch (error) {
+                    throw new PlaceError("DIRS_REFUSED", said(`making ${dir.slice(run.length)} in the run over the control was refused: ${error?.message ?? error}`), { cause: error });
+                }
+            }
+        };
+        const fetchedSizes = new Map();
+        const carrying = new Map();
         const handleOf = new Map();
         const mint = async (attempt) => {
             const t = await mintMountToken(control, { run: { id: ref.id, attempt }, prefix: options.ensure?.tokenPrefix, ttl: options.ensure?.tokenTtl ?? TOKEN_TTL });
@@ -385,7 +692,11 @@ export function archilPlace(options) {
             const m = given && !given.removed ? given : await mint(order.gen);
             // The driver gets the order's environment with each start; everything else is the host's own.
             const driver = {
-                start: (r, token, attempt) => host.start(r, token, { attempt: attempt?.attempt ?? order.gen, env: order.env }),
+                // Inside ensureRunning, after its revoke and start mark: no delegation holds the run, so S3 takes the writes.
+                start: async (r, token, attempt) => {
+                    await makeDirs(order.gen);
+                    return host.start(r, token, { attempt: attempt?.attempt ?? order.gen, env: order.env });
+                },
                 status: (h) => host.status(h),
                 stop: (h) => host.stop(h),
             };
@@ -414,6 +725,7 @@ export function archilPlace(options) {
                     if (result.adopted)
                         await drop(m).catch(() => { });
                     handles.push(result.handle);
+                    started.push(result.handle);
                     const w = watch(result.handle);
                     const mark = result.startMark ? await readStartMark(control, ref.id).catch(() => null) : null;
                     const runner = { epoch: mark?.generation ?? order.gen, exited: w.exited };
@@ -460,6 +772,26 @@ export function archilPlace(options) {
             paths,
             kills: host.kill && host.kills ? [host.kills] : [],
             token: async (attempt) => (await mint(attempt)).token,
+            ...(recording
+                ? {
+                    recordHome: {
+                        // In the run's directory, so the runner's delegation covers it and deleteRunTree removes it.
+                        dir: (attempt) => posix.join(root, RECORDS, String(countFrom1(attempt))),
+                        // One carry at a time per destination in this place; another place or process is held off by the file lock.
+                        bringHome: (into, carry = {}) => {
+                            const key = resolve(into);
+                            const run = (carrying.get(key) ?? Promise.resolve()).catch(() => { }).then(() => carryHome(control, records, into, carry.final === true, fetchedSizes));
+                            const settled = run.then(() => { }, () => { });
+                            carrying.set(key, settled);
+                            void settled.then(() => {
+                                if (carrying.get(key) === settled)
+                                    carrying.delete(key);
+                            });
+                            return run;
+                        },
+                    },
+                }
+                : {}),
             // The request goes in the run's directory, outside work/ where the agent's tools are confined, and the place reads
             // it over S3: a runner on a box of its own cannot reach the supervisor's files.
             requestFile: (attempt) => posix.join(root, REQUESTS, requestName(attempt)),
@@ -491,7 +823,8 @@ export function archilPlace(options) {
                         throw error;
                     }),
                 ]);
-                return { record, delegations: delegations.map(({ clientId, inodeId, path, isOrphaned, isPending }) => ({ clientId, inodeId, path, isOrphaned, isPending })) };
+                const boxes = host.box ? started.map((h) => host.box(h)).filter((b) => b !== null) : undefined;
+                return { record, delegations: delegations.map(({ clientId, inodeId, path, isOrphaned, isPending }) => ({ clientId, inodeId, path, isOrphaned, isPending })), ...(boxes ? { boxes } : {}) };
             },
             recover: async (last, kind, order) => {
                 if (killed.has(last))

package/dist/run.js

@@ -15,7 +15,7 @@ import { archilEnv } from "./env.js";
 import { EXIT_FENCED, FencedError, HeldError, ownWriteFenced, PdaError } from "./errors.js";
 import { openArchilStore } from "./store.js";
 import { LeaseWatchdog } from "./watchdog.js";
-import { currentHolder, DEFAULT_HEARTBEAT_MS, DEFAULT_LEASE_EXPIRY_MS, DEFAULT_LEASE_MARGIN_MS, openClaimDir, persistRecord, readRunRecord, RUN_JSON, RunRecordError, writeRunRecord, } from "./status.js";
+import { currentHolder, DEFAULT_HEARTBEAT_MS, DEFAULT_LEASE_EXPIRY_MS, DEFAULT_LEASE_MARGIN_MS, keepStartMark, openClaimDir, persistRecord, readRunRecord, RUN_JSON, RunRecordError, writeRunRecord, } from "./status.js";
 /** The same-host owner lock under the run's root. */
 export const OWNER_LOCK = "owner.lock";
 /** pi's store under the claim's `store/` directory. */
@@ -358,7 +358,16 @@ class Lease {
         if (seed !== undefined && previous === undefined) {
             await step("seed", async () => {
                 await this.#dir.assertMounted();
-                await seed({ root: claim.root, storePath: join(claim.store, STORE_FILE), workDir: claim.work });
+                // While a new run seeds it has no `run.json` heartbeat, and a supervisor keeps a start in its grace only while its
+                // start mark is younger than the grace: the mark is kept fresh until the seed ends, through the held claim directory.
+                // A seeder that dies stops, and the grace runs out as for any start.
+                const stop = keepStartMark(this.#dir.at, this.#lease.heartbeatMs, claim.root);
+                try {
+                    await seed({ root: claim.root, storePath: join(claim.store, STORE_FILE), workDir: claim.work });
+                }
+                finally {
+                    await stop();
+                }
                 await this.#dir.assertMounted();
                 await claim.barrier();
             });

package/dist/status.d.ts

@@ -81,6 +81,29 @@ export declare const persistRecord: PersistRecord;
  * `PersistRecord`.
  */
 export declare function writeRunRecord(root: string | ClaimDir, record: RunRecord, persist?: PersistRecord, signal?: AbortSignal): Promise<void>;
+/**
+ * The supervisor's record of its last start of a run, in the run's directory: the generation the instance will write,
+ * when, by whom, and the failed starts right before it (`failures`; `lastExit`, what the decision saw of the last one).
+ */
+export declare const START_MARK = "start.json";
+export type StartMark = {
+    generation: number;
+    at: string;
+    by: string | null;
+    failures: number;
+    lastExit: string | null;
+};
+/** A start mark's text as a mark, or null when it is not one (an unreadable mark gives no grace). */
+export declare function parseStartMark(text: string): StartMark | null;
+/**
+ * Keeps a supervisor's start mark for generation 1 in `dir` fresh: its `at` rewritten every `everyMs` (temp file, fsync,
+ * rename, fsync the directory), the rest of it as the supervisor wrote it; nothing is made when there is none. On a thread of
+ * its own, so a caller whose thread is blocked (a synchronous import) is kept fresh too. Only while `dir` is the directory
+ * the keeper first saw there. Best effort: a failure leaves the mark as it was, and the first one is a process warning
+ * (`PDA_START_MARK`) naming `label`; the grace then runs out as for any start. Returns the stop, which resolves once the
+ * thread has exited, so no write outlives it.
+ */
+export declare function keepStartMark(dir: string, everyMs: number, label?: string): () => Promise<void>;
 /** The FencedError code of a claim root that is no longer the mount the instance opened (unmounted, lazily or not). */
 export declare const CLAIM_UNMOUNTED = "CLAIM_UNMOUNTED";
 /** The file system type of an Archil mount in the mount table. */

package/dist/status.js

@@ -6,7 +6,9 @@ import { constants } from "node:fs";
 import { open, readFile, rename, rm, stat } from "node:fs/promises";
 import { hostname } from "node:os";
 import { join } from "node:path";
+import { Worker } from "node:worker_threads";
 import { FencedError, ownWriteFenced, PdaError } from "./errors.js";
+import { fdPath, hold } from "./held-fd.js";
 import { thisBoot } from "./process-table.js";
 export const RUN_JSON = "run.json";
 /** The temporary file every write goes through; renamed over `run.json` once it is fsynced. */
@@ -158,6 +160,122 @@ export async function writeRunRecord(root, record, persist = persistRecord, sign
         throw ownWriteFenced(join(typeof root === "string" ? root : root.root, RUN_JSON), error);
     }
 }
+// ---- the supervisor's start mark --------------------------------------------------------------------------------------
+/**
+ * The supervisor's record of its last start of a run, in the run's directory: the generation the instance will write,
+ * when, by whom, and the failed starts right before it (`failures`; `lastExit`, what the decision saw of the last one).
+ */
+export const START_MARK = "start.json";
+/** A start mark's text as a mark, or null when it is not one (an unreadable mark gives no grace). */
+export function parseStartMark(text) {
+    try {
+        const v = JSON.parse(text);
+        if (!Number.isSafeInteger(v.generation) || v.generation < 1 || typeof v.at !== "string" || !Number.isFinite(Date.parse(v.at)))
+            return null;
+        const failures = Number.isSafeInteger(v.failures) && v.failures >= 0 ? v.failures : 0;
+        const lastExit = typeof v.lastExit === "string" ? v.lastExit : null;
+        return { generation: v.generation, at: v.at, by: typeof v.by === "string" ? v.by : null, failures, lastExit };
+    }
+    catch {
+        return null;
+    }
+}
+// The keeper's code is plain CommonJS in a string, started with `eval: true` and no inherited Node flags, as the lease
+// watchdog's is (src/watchdog.ts says why). One refresh at a time, all synchronous: a stop (the message) is handled only
+// between two refreshes, so none is still writing once the thread has exited. It writes only while `dir` is still the
+// directory it first saw (its device and inode, read on this thread, so a stalled mount never holds the caller's): a path
+// that names another (a descriptor number reused) or none is never written, and the keeper stops. Its first failure goes
+// to the main thread, once.
+const KEEPER_SOURCE = `"use strict";
+const fs = require("node:fs");
+const path = require("node:path");
+const { parentPort, workerData } = require("node:worker_threads");
+const { dir, name, everyMs } = workerData;
+const file = path.join(dir, name);
+let first;
+let stopped = false;
+let reported = false;
+const report = (message) => {
+  if (reported) return;
+  reported = true;
+  parentPort.postMessage(message);
+};
+function refresh() {
+  if (stopped) return;
+  let now;
+  try {
+    now = fs.statSync(dir, { bigint: true });
+  } catch (error) {
+    stopped = true;
+    return report("its directory cannot be reached (" + (error.code || error.message) + "), so it is no longer the directory the keeper started on");
+  }
+  first ??= { dev: now.dev, ino: now.ino };
+  if (now.dev !== first.dev || now.ino !== first.ino) {
+    stopped = true;
+    return report("its directory is no longer the directory the keeper started on");
+  }
+  let text;
+  try {
+    text = fs.readFileSync(file, "utf8");
+  } catch (error) {
+    if (error.code === "ENOENT") return; // no mark: nothing to keep
+    return report("reading it failed (" + (error.code || error.message) + ")");
+  }
+  let mark;
+  try {
+    mark = JSON.parse(text);
+  } catch {
+    return; // not a mark: nothing to keep
+  }
+  if (mark === null || typeof mark !== "object" || mark.generation !== 1 || typeof mark.at !== "string" || !Number.isFinite(Date.parse(mark.at))) return;
+  try {
+    const temp = file + ".tmp";
+    const fd = fs.openSync(temp, "w", 0o644);
+    try {
+      fs.writeSync(fd, JSON.stringify({ ...mark, at: new Date().toISOString() }));
+      fs.fsyncSync(fd);
+    } finally {
+      fs.closeSync(fd);
+    }
+    fs.renameSync(temp, file);
+    const handle = fs.openSync(dir, "r");
+    try {
+      fs.fsyncSync(handle);
+    } finally {
+      fs.closeSync(handle);
+    }
+  } catch (error) {
+    report("rewriting it failed (" + (error.code || error.message) + ")");
+  }
+}
+const timer = setInterval(refresh, everyMs);
+refresh();
+parentPort.once("message", () => {
+  clearInterval(timer);
+  parentPort.close();
+});
+`;
+/**
+ * Keeps a supervisor's start mark for generation 1 in `dir` fresh: its `at` rewritten every `everyMs` (temp file, fsync,
+ * rename, fsync the directory), the rest of it as the supervisor wrote it; nothing is made when there is none. On a thread of
+ * its own, so a caller whose thread is blocked (a synchronous import) is kept fresh too. Only while `dir` is the directory
+ * the keeper first saw there. Best effort: a failure leaves the mark as it was, and the first one is a process warning
+ * (`PDA_START_MARK`) naming `label`; the grace then runs out as for any start. Returns the stop, which resolves once the
+ * thread has exited, so no write outlives it.
+ */
+export function keepStartMark(dir, everyMs, label = dir) {
+    const warn = (why) => process.emitWarning(`the start mark of ${label} is no longer kept fresh: ${why}; a supervisor's start grace runs out as for any start`, { code: "PDA_START_MARK" });
+    const worker = new Worker(KEEPER_SOURCE, { eval: true, workerData: { dir, name: START_MARK, everyMs }, execArgv: [] });
+    worker.on("message", (why) => warn(why));
+    worker.on("error", (error) => warn(`its thread failed (${error.message})`));
+    worker.unref();
+    const exited = new Promise((resolve) => worker.once("exit", () => resolve()));
+    return () => {
+        worker.ref();
+        worker.postMessage("stop");
+        return exited;
+    };
+}
 // ---- the claim directory ----------------------------------------------------------------------------------------------
 /** The FencedError code of a claim root that is no longer the mount the instance opened (unmounted, lazily or not). */
 export const CLAIM_UNMOUNTED = "CLAIM_UNMOUNTED";
@@ -216,7 +334,8 @@ export async function openClaimDir(root, options = {}) {
         throw error instanceof FencedError ? error : unmounted(`checking the claim root ${root} failed: ${error.message}`, error);
     }
     const byPath = options.byPath ?? process.platform === "darwin";
-    const at = byPath ? root : `/proc/self/fd/${handle.fd}`;
+    // Held until close(): `at` names the directory by its number, and a collection must never free that number.
+    const at = byPath ? (hold(handle), root) : fdPath(handle);
     let closed = false;
     const same = (path, now, claims) => {
         if (now.dev === claims.dev && now.ino === claims.ino)