@parcha/pi-durable-disk 0.1.0-beta.12 → 0.1.0-beta.14

package/dist/claim.js

@@ -195,18 +195,89 @@ async function runDirsByInode(control, inodes) {
     }
     return names;
 }
-/** Revoke every delegation on the run through the control API. The old holder's next fsync returns EIO. */
+/** A client's private directories on the disk: `.archil/client-<clientId>` and its `unlinked/`. */
+const PRIVATE_DIR = /^\/*\.archil\/client-([A-Za-z0-9_-]+)(\/unlinked)?$/;
+/**
+ * The delegations `held`'s clients also hold on their own private directories (`privateDirs`: inode to the client whose
+ * private directory it is). A killed client leaves them orphaned, and nothing ties them to a run: they block no run, but
+ * they pile up and resurface in listings. A client that holds anything else beyond `held` may hold another run, so none
+ * of its delegations is a companion.
+ */
+export function companionsOf(all, held, privateDirs) {
+    const clients = new Set(held.map((d) => d.clientId));
+    const listed = new Set(held.map((d) => `${d.clientId}/${d.inodeId}`));
+    const others = all.filter((d) => clients.has(d.clientId) && !listed.has(`${d.clientId}/${d.inodeId}`));
+    const own = (d) => (d.path ? PRIVATE_DIR.exec(d.path)?.[1] : privateDirs.get(d.inodeId)) === d.clientId;
+    const holdsMore = new Set(others.filter((d) => !own(d)).map((d) => d.clientId));
+    return others.filter((d) => !holdsMore.has(d.clientId));
+}
+/** The inodes of `clients`' private directories, by one `exec` (`stat`); none without `exec`. */
+async function privateDirInodes(control, clients) {
+    const asked = clients.filter((c) => /^[A-Za-z0-9_-]+$/.test(c));
+    const dirs = new Map();
+    if (!control.exec || asked.length === 0)
+        return dirs;
+    const paths = asked.flatMap((c) => [`.archil/client-${c}`, `.archil/client-${c}/unlinked`]);
+    const r = await control.exec(`stat -c '%i %n' -- ${paths.join(" ")} 2>/dev/null; true`);
+    for (const line of String(r.stdout ?? "").split("\n")) {
+        const m = /^(\d+) (\S+)$/.exec(line);
+        const client = m && PRIVATE_DIR.exec(m[2])?.[1];
+        if (m && client && asked.includes(client))
+            dirs.set(Number(m[1]), client);
+    }
+    return dirs;
+}
+/**
+ * Revoke the companions of `held` (`companionsOf`): only for clients the caller is already revoking because it fences or
+ * deletes a run, never as a sweep. From `all` or a fresh listing. Best effort, and hygiene only: a companion blocks no
+ * run, so one left behind (or a failed listing or `exec`) costs a listing entry. Returns the ones revoked.
+ */
+export async function revokeCompanions(control, held, all) {
+    if (held.length === 0)
+        return [];
+    const listing = all ?? (await control.listDelegations().catch(() => []));
+    const clients = [...new Set(held.map((d) => d.clientId))];
+    const listed = new Set(held.map((d) => `${d.clientId}/${d.inodeId}`));
+    if (!listing.some((d) => clients.includes(d.clientId) && !listed.has(`${d.clientId}/${d.inodeId}`)))
+        return [];
+    const dirs = await privateDirInodes(control, clients).catch(() => new Map());
+    const revoked = [];
+    for (const d of companionsOf(listing, held, dirs)) {
+        await control.revokeDelegation({ clientId: d.clientId, inodeId: d.inodeId }).then(() => revoked.push(d), () => { });
+    }
+    return revoked;
+}
+/**
+ * Revoke every delegation on the run through the control API, then its holders' private directories
+ * (`revokeCompanions`). The old holder's next fsync returns EIO. Returns the run's own delegations.
+ */
 export async function revoke(control, id) {
     const path = runPath(id);
+    let all;
+    let held;
     try {
-        const held = await findDelegations(control, id);
+        all = await control.listDelegations();
+        held = await matchDelegations(all, id, pathlessResolver(control));
         for (const d of held)
             await control.revokeDelegation(d);
-        return held;
     }
     catch (err) {
         throw new ClaimError("CONTROL_API_FAILED", `revoking ${path} failed`, { cause: err });
     }
+    await revokeCompanions(control, held, all);
+    return held;
+}
+/**
+ * Revoke what `revoke` revokes, skipping every revoke that fails: for cleanup, where deleting the tree next checks what
+ * is left. Only the listing (and a pathless resolution) can fail it. Returns the run's own delegations.
+ */
+export async function revokeBestEffort(control, id) {
+    const all = await control.listDelegations();
+    const held = await matchDelegations(all, id, pathlessResolver(control));
+    for (const d of held)
+        await control.revokeDelegation({ clientId: d.clientId, inodeId: d.inodeId }).catch(() => { });
+    await revokeCompanions(control, held, all);
+    return held;
 }
 /** Revoke through the control API, then mount plainly; if the API path fails, mount with `--force`. */
 export async function takeOver(control, opts) {
@@ -297,14 +368,24 @@ function isOurs(entry, ref) {
     const base = `${ref.disk}:/${runPath(ref.id)}`;
     return entry.fstype === "fuse.archil" && (entry.source === base || entry.source === `${base}[${ref.region}]`);
 }
-/** A FUSE mount whose daemon is gone answers stat with ENOTCONN; `archil` refuses it as "not running". */
+/**
+ * The errno codes of a stat on a FUSE mount whose daemon is gone: ENOTCONN once its connection is torn down, ECONNABORTED
+ * for a stat already in flight while it is torn down. Codes, never messages: a message is translated by the locale.
+ */
+export const DEAD_CONNECTION = new Set(["ENOTCONN", "ECONNABORTED"]);
+/**
+ * Whether the FUSE daemon behind a mount is gone, which `archil` reports as "not running". Its connection is torn down,
+ * so a stat fails with ENOTCONN; a stat already in flight while the dying daemon's connection is torn down (right after a
+ * kill -9, while its threads exit) fails with ECONNABORTED instead. Both are the connection's own end: a live daemon,
+ * however slow, answers, and a stopped one keeps the stat waiting.
+ */
 async function isDead(host, mountpoint) {
     try {
         await host.stat(mountpoint);
         return false;
     }
     catch (err) {
-        return err.code === "ENOTCONN";
+        return DEAD_CONNECTION.has(String(err.code));
     }
 }
 // Exit codes of the mount tools are not evidence (a busy mount can report success and stay); the mount table decides.

package/dist/cli.d.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import { type Parking } from "./park.js";
 import { type RunServer } from "./serve.js";
+import { type ArchilControl, type DiskClient } from "./supervise.js";
 import { type DurableRun } from "./run.js";
 /** "30s", "500ms", "2m", "1h", or plain milliseconds. */
 /**
@@ -28,6 +29,11 @@ export declare function runUser(spec: string, passwd?: string): {
     home: string;
     name: string;
 };
+/**
+ * The disk's control API for the supervisor (`archilControl`), from the API key in `--api-key-env` (default
+ * ARCHIL_API_KEY) and `--disk` and `--region`. `client` replaces the `disk` client (tests).
+ */
+export declare function cliControl(values: Record<string, unknown>, client?: DiskClient): Promise<ArchilControl>;
 /** What the instance does besides the app between its open and its exit. */
 export interface InstanceLife {
     /** How long a drain waits for running work before the release; 0 (default) releases at once. */

package/dist/cli.js

@@ -9,7 +9,7 @@
 // The API key is read from the environment variable named by `--api-key-env` (default ARCHIL_API_KEY) and only by the
 // supervisor commands; `run` never needs it and a host driver never passes it on.
 import { closeSync, openSync, readFileSync, realpathSync, statSync } from "node:fs";
-import { join, resolve } from "node:path";
+import { basename, join, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 import { parseArgs } from "node:util";
 import { ARCHIL_SCOPED, DEFAULT_MOUNT_ROOT, runPath, TOKEN_PREFIX, unmountClaim } from "./claim.js";
@@ -19,8 +19,11 @@ import { fork } from "./fork.js";
 import { drain, recordWake, watchParking } from "./park.js";
 import { isLoopback, readServeToken, serveRun } from "./serve.js";
 import { dockerHost } from "./hosts/docker.js";
-import { localHost } from "./hosts/local-host.js";
-import { checkHost, CONTROL_TIMEOUT_MS, readRunStatus, superviseRuns, sweepTokens, withTimeouts, } from "./supervise.js";
+import { localHost, procStartTicks } from "./hosts/local-host.js";
+import { processTable, thisBoot } from "./process-table.js";
+import { localRun } from "./local-run.js";
+import { parseRunRecord, RUN_JSON } from "./status.js";
+import { archilControl, checkHost, CONTROL_TIMEOUT_MS, readRunStatus, superviseRuns, sweepTokens, withTimeouts, } from "./supervise.js";
 import { openDurableRun } from "./run.js";
 import { AppError, loadApp } from "./app.js";
 class UsageError extends PdaError {
@@ -29,6 +32,7 @@ class UsageError extends PdaError {
     }
 }
 const USAGE = `usage:
+  pi-durable-disk run --dir PATH --app MODULE [--id ID] [any run flag below but the Archil ones]
   pi-durable-disk run --disk D --region R --id ID --app MODULE [--mount-root DIR] [--archil BIN]
                         [--heartbeat-ms N] [--lease-expiry-ms N] [--lease-margin-ms N] [--on-sigterm resume|pause]
                         [--token-stdin] [--run-as USER] [--serve PORT] [--serve-host H] [--serve-url URL]
@@ -42,6 +46,7 @@ const USAGE = `usage:
                         with --host docker: --image IMAGE [--app-root DIR] [--fleet F] [--name-prefix P] [--run-as USER]
                         [--docker-arg ARG ...] [--apparmor auto|PROFILE|none] [--docker BIN]
   pi-durable-disk supervise --check --disk D --region R [--mount-root DIR] [--user U] [--group G] [--check-id-prefix P]
+  pi-durable-disk status --dir PATH
   pi-durable-disk status --disk D --region R --id ID [--host-name NAME] [--api-key-env NAME] [--docker BIN]
   pi-durable-disk release --id ID [--mount-root DIR]
   pi-durable-disk fork --disk D --region R --id ID --new-id NEW [--mount-root DIR] [--archil BIN] [--token-prefix P]
@@ -141,29 +146,16 @@ const need = (v, ...keys) => {
 };
 const str = (v, fallback) => (typeof v === "string" ? v : fallback);
 const emit = (line) => process.stdout.write(`${JSON.stringify({ at: new Date().toISOString(), ...line })}\n`);
-/** The disk's control API for the supervisor, plus the two lists only the disk record carries (users, clients). */
-async function control(values) {
+/**
+ * The disk's control API for the supervisor (`archilControl`), from the API key in `--api-key-env` (default
+ * ARCHIL_API_KEY) and `--disk` and `--region`. `client` replaces the `disk` client (tests).
+ */
+export async function cliControl(values, client) {
     const name = str(values["api-key-env"], "ARCHIL_API_KEY");
     const apiKey = process.env[name];
     if (!apiKey)
         throw new UsageError(`no API key in $${name}`);
-    const { configure, getDisk } = await import("disk");
-    configure({ apiKey, region: str(values.region) });
-    const id = str(values.disk);
-    const disk = await getDisk(id);
-    return {
-        getObject: (key) => disk.getObject(key),
-        headObject: (key) => disk.headObject(key),
-        putObject: (key, body, options) => disk.putObject(key, body, options),
-        addUser: (user) => disk.addUser(user),
-        removeUser: (type, identifier) => disk.removeUser(type, identifier),
-        listDelegations: () => disk.listDelegations(),
-        revokeDelegation: (d) => disk.revokeDelegation(d),
-        exec: (command) => disk.exec(command),
-        listObjects: (prefix, options) => disk.listObjects(prefix, options),
-        deleteObjects: (keys, options) => disk.deleteObjects(keys, options),
-        listUsers: async () => (await getDisk(id)).authorizedUsers ?? [],
-    };
+    return archilControl({ apiKey, disk: str(values.disk), region: str(values.region), ...(client ? { client } : {}) });
 }
 // ---- run ---------------------------------------------------------------------------------------------------------------
 /**
@@ -201,8 +193,12 @@ async function readToken(values) {
  * between the open and the exit is `serveUntilDone`.
  */
 async function runInstance(values) {
-    need(values, "disk", "region", "id", "app");
-    const ref = { disk: str(values.disk), region: str(values.region), id: str(values.id) };
+    const dir = localDir(values, ["disk", "region", "mount-root", "archil", "token-stdin"]);
+    if (dir === undefined)
+        need(values, "disk", "region", "id", "app");
+    else
+        need(values, "app");
+    const ref = dir === undefined ? { disk: str(values.disk), region: str(values.region), id: str(values.id) } : { disk: "local", region: "local", id: str(values.id) ?? basename(dir) };
     const runAs = typeof values["run-as"] === "string" ? runUser(values["run-as"]) : undefined;
     if (runAs && process.getuid?.() !== 0)
         throw new UsageError("--run-as needs an instance that runs as root (inside a container); elsewhere run the instance as the run user");
@@ -222,7 +218,7 @@ async function runInstance(values) {
         throw new UsageError(`--serve-host ${serveHost} is not loopback: name a --serve-token-file`);
     const serveToken = tokenFile === undefined ? undefined : readServeToken(tokenFile);
     const mountRoot = str(values["mount-root"], DEFAULT_MOUNT_ROOT);
-    const root = join(mountRoot, runPath(ref.id));
+    const root = dir ?? join(mountRoot, runPath(ref.id));
     const log = (event, extra = {}) => process.stderr.write(`${JSON.stringify({ at: new Date().toISOString(), event, run: ref.id, ...extra })}\n`);
     let app;
     try {
@@ -232,8 +228,12 @@ async function runInstance(values) {
         log("app failed", { code: err.code, message: err.message });
         return exitCodeFor(err);
     }
-    const { onOpen, root: rootOptions, wake: appWake, ...harness } = app;
+    const { onOpen, beforeResume, root: rootOptions, wake: appWake, ...harness } = app;
     const holder = JSON.parse(process.env.PDA_HOLDER ?? "{}");
+    // On a local directory nothing else knows this instance: its start ticks go into the holder, so `status --dir` tells it
+    // from a later process that reuses its pid (a driver's handle already carries them).
+    if (dir !== undefined && typeof holder.startTicks !== "number")
+        holder.startTicks = procStartTicks(process.pid);
     const cgroup = watchdogOwnsCgroup(holder);
     if (cgroup.warning)
         log("cgroup not owned", { warning: cgroup.warning });
@@ -250,9 +250,8 @@ async function runInstance(values) {
     let run;
     try {
         run = await openDurableRun(ref, {
-            mountToken: await readToken(values),
-            mountRoot,
-            host: { archil: str(values.archil) },
+            // A local directory is its own claim (no mount, no token); a disk's run is mounted with a token.
+            ...(dir === undefined ? { mountToken: await readToken(values), mountRoot, host: { archil: str(values.archil) } } : localRun(dir)),
             harness,
             ...(runAs
                 ? { env: (c) => archilEnv(c, { runAs: { uid: runAs.uid, gid: runAs.gid }, shellEnv: { HOME: runAs.home, USER: runAs.name, LOGNAME: runAs.name } }) }
@@ -261,6 +260,7 @@ async function runInstance(values) {
             ownCgroup: cgroup.owns,
             lease: Object.fromEntries(Object.entries(lease).filter(([, v]) => v !== undefined)),
             onStep: (step, t) => void (steps[step] = Math.round(t)),
+            ...(beforeResume ? { beforeResume } : {}),
         });
     }
     catch (err) {
@@ -450,7 +450,7 @@ async function supervise(values) {
         throw new UsageError("--check proves this host's own mounts; with --host docker the containers mount, so it does not apply");
     if (docker && (typeof values.image !== "string" || !values.image))
         throw new UsageError("--host docker needs --image");
-    const disk = await control(values);
+    const disk = await cliControl(values);
     const controlTimeoutMs = values["control-timeout"] ? parseDuration(str(values["control-timeout"])) : CONTROL_TIMEOUT_MS;
     if (values.check) {
         const report = await checkHost({
@@ -536,8 +536,11 @@ async function supervise(values) {
     return 0;
 }
 async function status(values) {
+    const dir = localDir(values, ["disk", "region", "mount-root", "api-key-env", "host-name", "docker"]);
+    if (dir !== undefined)
+        return localStatus(dir);
     need(values, "disk", "region", "id");
-    const disk = await control(values);
+    const disk = await cliControl(values);
     const id = str(values.id);
     const run = await readRunStatus(disk, id);
     const { findDelegations } = await import("./claim.js");
@@ -555,6 +558,67 @@ async function status(values) {
     emit({ run: id, runJson: run, delegations: delegations.map(({ clientId, inodeId, path, isOrphaned, isPending }) => ({ clientId, inodeId, path, isOrphaned, isPending })), holderStatus, ...(container ? { container } : {}) });
     return 0;
 }
+/**
+ * `--dir PATH`: the run is the local directory PATH (`localRun`), so the Archil flags mean nothing and are refused. Returns
+ * the directory, absolute; undefined without `--dir`.
+ */
+function localDir(values, archilFlags) {
+    if (values.dir === undefined)
+        return undefined;
+    const dir = str(values.dir);
+    if (!dir)
+        throw new UsageError("--dir needs a path");
+    const mixed = archilFlags.filter((flag) => values[flag] !== undefined);
+    if (mixed.length > 0)
+        throw new UsageError(`--dir is a run on a local directory: it takes no ${mixed.map((flag) => `--${flag}`).join(", ")}`);
+    return resolve(dir);
+}
+/**
+ * `status --dir PATH`: `run.json` read from the directory, and whether its holder is still the process that claimed it:
+ * `running` when it is this boot's and its pid is alive with the start ticks the holder recorded (a later process that
+ * reuses the pid has others), `stopped` when it is not, `gone` from another boot, and `unknown` when the holder recorded no
+ * start ticks (a run opened by a library caller) and its pid is alive.
+ */
+function localStatus(dir) {
+    let text;
+    try {
+        text = readFileSync(join(dir, RUN_JSON), "utf8");
+    }
+    catch (err) {
+        throw new UsageError(`no run in ${dir}: ${err.message}`);
+    }
+    const run = parseRunRecord(text);
+    let holderStatus = null;
+    const holder = run.holder;
+    if (holder) {
+        // A read that failed (sysctl or ps on macOS) is no exit: it says `unknown`, never `gone` or `stopped`.
+        const boot = thisBoot();
+        const ticks = startOf(Number(holder.pid));
+        if (boot === null)
+            holderStatus = "unknown";
+        else if (holder.bootId !== boot)
+            holderStatus = "gone";
+        else if (ticks === undefined)
+            holderStatus = "unknown";
+        else if (typeof holder.startTicks === "number")
+            holderStatus = ticks === holder.startTicks ? "running" : "stopped";
+        else
+            holderStatus = ticks === null ? "stopped" : "unknown";
+    }
+    emit({ run: run.run, dir, runJson: run, holderStatus });
+    return 0;
+}
+/** A live process's start (null: no such process, or a zombie), or undefined when the process table cannot be read. */
+function startOf(pid) {
+    if (!Number.isInteger(pid) || pid <= 1)
+        return null;
+    try {
+        return processTable().snapshot().info(pid)?.start ?? null;
+    }
+    catch {
+        return undefined;
+    }
+}
 async function release(values) {
     need(values, "id");
     const via = await unmountClaim(join(str(values["mount-root"], DEFAULT_MOUNT_ROOT), runPath(str(values.id))));
@@ -563,7 +627,7 @@ async function release(values) {
 }
 async function forkRun(values) {
     need(values, "disk", "region", "id", "new-id");
-    const disk = await control(values);
+    const disk = await cliControl(values);
     const result = await fork({ disk: str(values.disk), region: str(values.region), id: str(values.id) }, str(values["new-id"]), {
         control: disk,
         mountRoot: str(values["mount-root"], DEFAULT_MOUNT_ROOT),
@@ -582,6 +646,7 @@ export async function main(argv) {
             case "run":
                 return await runInstance(parse(rest, {
                     ...COMMON,
+                    dir: { type: "string" },
                     archil: { type: "string" },
                     app: { type: "string" },
                     "heartbeat-ms": { type: "string" },
@@ -640,7 +705,7 @@ export async function main(argv) {
                     apparmor: { type: "string" },
                 }));
             case "status":
-                return await status(parse(rest, { ...COMMON, docker: { type: "string" } }));
+                return await status(parse(rest, { ...COMMON, dir: { type: "string" }, docker: { type: "string" } }));
             case "release":
                 return await release(parse(rest, COMMON));
             case "fork":

package/dist/commands.d.ts

@@ -0,0 +1,70 @@
+import { type ProcessTable } from "./process-table.js";
+/** The environment variable that marks a command as one environment's: its value is that recorder's token. */
+export declare const COMMANDS_MARK = "PI_DURABLE_DISK_COMMANDS";
+/** One process, as recorded: a pid alone could be reused, a pid with its start on one boot cannot (process-table.ts). */
+export interface ProcessMark {
+    readonly pid: number;
+    readonly start: number;
+}
+/** What a record holds: a command's group leader (null once only members of it outlived a reap) and the descendants seen. */
+export interface CommandRecord {
+    readonly boot: string;
+    readonly leader: ProcessMark | null;
+    readonly members: readonly ProcessMark[];
+}
+/** What a reap did: how many processes it signalled, and those still alive at its deadline (kept for the next open). */
+export interface Reaped {
+    readonly signalled: number;
+    readonly survivors: readonly ProcessMark[];
+}
+export interface CommandsOptions {
+    /** The `/proc` to read; for tests. */
+    readonly proc?: string;
+    /** The process table to read; default this machine's (`/proc`, or `proc` when given; ps on macOS). For tests. */
+    readonly table?: ProcessTable;
+    /** How a reap signals; default `process.kill`. For tests. */
+    readonly kill?: (target: number, signal: NodeJS.Signals) => void;
+}
+/**
+ * Records this environment's command groups under this process while commands run. `started()` and `finished()` bracket
+ * each command, which carries `COMMANDS_MARK=<token>` in its environment; while any runs, the tree is walked soon after a
+ * start and then every `everyMs`. A record is written when its group gains a process and removed once nothing in it is
+ * alive. A walk that fails (a record cannot be written: a full disk) goes to `onError` the first time, never out of a
+ * timer, and the next walk tries again.
+ */
+export declare class CommandRecorder {
+    #private;
+    /** The value of `COMMANDS_MARK` in this environment's commands. */
+    readonly token: `${string}-${string}-${string}-${string}-${string}`;
+    constructor(dir: string, options?: CommandsOptions & {
+        readonly instance?: number;
+        readonly everyMs?: number;
+        readonly onError?: (error: unknown) => void;
+    });
+    /** A command is about to start. */
+    started(): void;
+    /** A command ended (its result is in). The last one stops the walks after a final one. */
+    finished(): void;
+    /** Stops recording (the environment closed); the records stay for the next open. */
+    close(): void;
+    /** One walk. It is called from timers, so it never throws: a failure goes to `onError` once, and the next walk retries. */
+    scan(): void;
+}
+/**
+ * Ends what a previous incarnation's commands left running: for each record of this boot, the leader's whole group when
+ * the leader is still the recorded process, and each recorded descendant that still is. Waits up to `timeoutMs` until
+ * none of them is alive. A record whose processes are all gone is removed; one that still names a live process (it did
+ * not die at the deadline: a process in an uninterruptible wait) is rewritten with just those, so the next open tries
+ * again, and they are returned. Run only while holding the run (after its owner lock), before the Harness opens.
+ */
+export declare function reapCommands(dir: string, options?: CommandsOptions & {
+    readonly timeoutMs?: number;
+}): Promise<Reaped>;
+/**
+ * `reapCommands` at an open: refuses the run (`HeldError("commands")`, exit 76) when a process a killed incarnation's
+ * commands left is still alive after it, since the run must not resume beside it. Its record stays, and the next open
+ * tries again.
+ */
+export declare function reapOrRefuse(dir: string, options?: CommandsOptions & {
+    readonly timeoutMs?: number;
+}): Promise<Reaped>;