@platless/axom-cli 0.3.13 → 0.5.9

package/dist/workstation/exec/policy.js

@@ -0,0 +1,837 @@
+// The daemon's exec POLICY (spec §4.2, §8.2, §14; analysis E §3.1, §3.6, §6.3–6.4) on top of keeper-terminal's exec mechanism:
+// every `exec_request` becomes `[shell, '-lc', command]` with the daemon-built environment and a deadline from the daemon's own
+// clock, runs in keeper-terminal when it is available and compatible (durable: it survives a daemon restart), else in this process
+// (the fallback, not durable), at most 32 at a time. Results are shaped by bytes (head + tail, UTF-8 safe). Cancel = SIGTERM, 5 s,
+// SIGKILL. A restarted daemon recovers the execs it finds in keeper-terminal from their checkpoints.
+//
+// Wire ≤ 4: today's COMPUTER frames unchanged (exec_request → streamed exec_output → exec_exit with the same fields); an exec dies
+// with its channel, so a keeper exec is killed on channel close and counts as a shell task exactly like an in-process one (`run`).
+// Wire ≥ 5 (`keeperExecsOutliveTheChannel`, spec §4.3, §8.2): the keeper exec id IS the server's exec id, an exec is owned by this
+// policy rather than by a channel (`startOrFind`/`find`/`cancel`/`acknowledge`; channels attach to it, exec/attach.ts), it is
+// never a shell task, and a restarted daemon keeps it for the server to re-attach.
+import { createHash, randomUUID } from 'node:crypto';
+import { KeeperFrameTooLargeError, KeeperNotConnectedError, KeeperReplacementRequiredError, KeeperRequestError } from '../keepers/kit/client.js';
+import { KeeperLaunchError } from '../keepers/terminal/client.js';
+import { EXEC_CAPABILITY, EXEC_SIGNALS, EXEC_TERMINAL_TTL_MS, ExecErrorCode } from '../keepers/terminal/protocol.js';
+import { execArgv, execCwd, execDeadline, execEnvironment } from './command.js';
+import { DurableExec, EXEC_KILL_GRACE_MS } from './durable.js';
+import { startExec as startInProcess } from './fallback.js';
+/** Policy cap on commands running at once (spec §14; keeper-terminal's own hard limit is 128). Before this task: unlimited. */
+export const EXEC_CONCURRENCY_LIMIT = 32;
+/** keeper-terminal keeps this much of each stream's beginning however much is dropped (covers the 32 KiB result head). */
+export const EXEC_RETAIN_HEAD_BYTES = 64 * 1024;
+/** Uncommitted output keeper-terminal keeps per exec before dropping the oldest of it. */
+export const EXEC_MAX_UNCOMMITTED_BYTES = 16 * 1024 * 1024;
+/** Activity label until the server sends one (wire v5 `label`): never the command line. */
+export const EXEC_DEFAULT_LABEL = 'Command';
+/** Answer timeout of exec.start and of the exec.status probe after an unanswered start (capped by the time left to the deadline). */
+export const EXEC_START_TIMEOUT_MS = 5000;
+const MIN_CALL_TIMEOUT_MS = 500;
+export const EXEC_LIMIT_MESSAGE = `Workstation is already running ${EXEC_CONCURRENCY_LIMIT} commands; wait for one to finish`;
+export const EXEC_UNCONFIRMED_MESSAGE = 'Workstation could not confirm whether the command started; it may be running. Check its effects before running it again.';
+/** The COMPUTER wire version that makes execs re-attachable (spec §4.3). */
+export const DURABLE_EXEC_WIRE_VERSION = 5;
+/**
+ * THE switch (spec §4.3 "compat", §8.2): does an exec running in keeper-terminal outlive the computer channel that started it?
+ *
+ * `false` under wire ≤ 4: the server treats a closed channel as the end of the exec and an agent may start the command again, so
+ * the daemon kills it on channel close (as it always did) and, since a daemon restart closes the channel, it counts in
+ * `activeShellTasks`. `true` from wire v5 on (exec_started/attach/ack): a keeper exec is never a shell task, is admitted while an
+ * update drains, survives a force update and a channel close, and a restarted daemon keeps it running for the server to re-attach.
+ */
+export function keeperExecsOutliveTheChannel(wireVersion) {
+    return wireVersion >= DURABLE_EXEC_WIRE_VERSION;
+}
+/**
+ * Prefix of the keeper exec ids of wire ≤ 4 execs (`run`): those die with their channel. Any other exec id is a wire v5 server id
+ * (`keeperExecIdOf`), which is how a restarted daemon tells the two apart when an exec has no checkpoint yet (it died between
+ * `exec.start` and the first checkpoint).
+ */
+export const CHANNEL_BOUND_EXEC_PREFIX = 'd4-';
+const channelBound = (execId) => execId.startsWith(CHANNEL_BOUND_EXEC_PREFIX);
+/** keeper-terminal's `jobId` bound (keepers/terminal/exec.ts): a longer one would make it refuse the start. */
+export const EXEC_JOB_ID_CHARS = 256;
+/** keeper-terminal's exec id grammar (keepers/terminal/exec.ts). */
+const KEEPER_EXEC_ID = /^[A-Za-z0-9_.:-]{1,128}$/;
+/**
+ * Wire v5: the keeper exec id of the server's exec id — the id itself (the server sends UUIDs), or a fixed hash of an id the keeper
+ * would not accept. Deterministic, so a restarted daemon finds the exec again from the server's id alone.
+ */
+export function keeperExecIdOf(serverId) {
+    const usable = KEEPER_EXEC_ID.test(serverId) && serverId !== '.' && serverId !== '..' && !channelBound(serverId);
+    return usable ? serverId : `h-${createHash('sha256').update(serverId).digest('hex').slice(0, 40)}`;
+}
+const isExecSignal = (value) => value !== undefined && EXEC_SIGNALS.includes(value);
+function errorCode(error) {
+    return typeof error === 'object' && error !== null && typeof error.code === 'string' ? error.code : undefined;
+}
+const messageOf = (error) => (error instanceof Error ? error.message : String(error));
+/** The keeper refused the request, or it never left this process: certainly nothing started. */
+function startedNothing(error) {
+    return error instanceof KeeperRequestError || error instanceof KeeperLaunchError || error instanceof KeeperNotConnectedError
+        || error instanceof KeeperFrameTooLargeError || error instanceof KeeperReplacementRequiredError;
+}
+const ABORTED = Symbol('aborted');
+/** `work`, or ABORTED as soon as `signal` aborts (work keeps going; the caller settles its late result). */
+function untilAborted(work, signal) {
+    if (!signal)
+        return work;
+    if (signal.aborted)
+        return Promise.resolve(ABORTED);
+    return new Promise((resolve, reject) => {
+        const onAbort = () => resolve(ABORTED);
+        signal.addEventListener('abort', onAbort, { once: true });
+        work.then((value) => { signal.removeEventListener('abort', onAbort); resolve(value); }, (error) => { signal.removeEventListener('abort', onAbort); reject(error); });
+    });
+}
+const exitFrame = (id, stderr, exitCode = null) => ({ t: 'exec_exit', id, exitCode, stdout: '', stderr, timedOut: false, truncated: false });
+/** Appended to stderr when keeper-terminal died while the command ran (its successor stopped the command or could not verify it). */
+export const KEEPER_LOST_NOTE = '\n[workstation] keeper-terminal exited while the command was running; the command did not finish normally.';
+export const KEEPER_STOPPED_NOTE = '\n[workstation] keeper-terminal was stopped while the command was running; the command was stopped.';
+export const unknownOutcomeNote = (reason) => `\n[workstation] Lost track of the command (${reason}); it may still be running or may have finished. Check its effects before running it again.`;
+/** The `exec_exit` frame of a durable outcome: today's field set, today's `exitCode:null` for timed out / cancelled / not finished. */
+export function durableExitFrame(id, outcome) {
+    let { stderr } = outcome;
+    if (outcome.state === 'unknown') {
+        return { t: 'exec_exit', id, exitCode: null, stdout: outcome.stdout, stderr: stderr + unknownOutcomeNote(outcome.reason), timedOut: false, truncated: outcome.truncated };
+    }
+    const { exit } = outcome;
+    if (exit.reason === 'spawn_failed')
+        stderr += exit.error ?? 'Command could not be started';
+    else if (exit.reason === 'keeper_lost')
+        stderr += KEEPER_LOST_NOTE;
+    else if (exit.reason === 'keeper_stopped' && !outcome.cancelled && !exit.timedOut)
+        stderr += KEEPER_STOPPED_NOTE;
+    const finished = !exit.timedOut && !outcome.cancelled && exit.reason === undefined;
+    return { t: 'exec_exit', id, exitCode: finished ? exit.code : null, stdout: outcome.stdout, stderr, timedOut: exit.timedOut, truncated: outcome.truncated };
+}
+/** How often `find` asks keeper-terminal before it reports the exec's state undecidable ({@link ExecStateUndecidableError}). */
+const FIND_ATTEMPTS = 3;
+/**
+ * Bound of one keeper-terminal presence probe in recovery and `find`: a real connect and hello. A keeper that does not answer within
+ * it is "could not tell", never "not there" (a loaded machine, a daemon that just restarted).
+ */
+export const EXEC_PROBE_TIMEOUT_MS = 3000;
+/**
+ * `find` could not tell whether keeper-terminal runs the exec (a keeper that does not answer, a listing that failed): the caller
+ * must not answer `unknown` — the exec may well be running — but ask again later (exec_attach defers its answer).
+ */
+export class ExecStateUndecidableError extends Error {
+    constructor(message) { super(message); this.name = 'ExecStateUndecidableError'; }
+}
+const PENDING_ACKS_LIMIT = 1024;
+/** Fallback execs remembered for re-attach (Task 26); ended ones are evicted first, oldest first. */
+export const FALLBACK_RECORDS_LIMIT = 1024;
+/**
+ * An ended fallback record keeps its exit (its shaped output, up to ~256 KiB) this long — far past the server's 90 s reconnect grace —
+ * and at most {@link FALLBACK_OUTPUT_RECORDS_LIMIT} ended records keep one at all. Past either it becomes a tombstone: still answered
+ * (exited, never "unknown", never run again), but with a short note instead of the output.
+ */
+export const FALLBACK_RECORD_TTL_MS = 10 * 60_000;
+export const FALLBACK_OUTPUT_RECORDS_LIMIT = 64;
+export const FALLBACK_EXPIRED_NOTE = '[workstation] This command ran in the workstation daemon (not durable); its output is no longer kept.';
+/** Exec ids this daemon was asked about (attach) or sent to keeper-terminal: a later request for one is a repeat (Task 26). */
+const SEEN_IDS_LIMIT = 4096;
+const FIND_RETRY_MS = 500;
+export class ExecPolicy {
+    options;
+    now;
+    log;
+    limit;
+    killGraceMs;
+    /** run() calls in progress (keeper or in-process). */
+    running = 0;
+    /** Durable execs this daemon follows outside a run(): detached ones and those recovered after a restart. */
+    followed = new Map();
+    /** Exec ids between `exec.start` and the run that follows them (recovery must not take them). */
+    starting = new Set();
+    inRun = new Set();
+    /** keeper-terminal processes whose execs were recovered (a keeper started later, or restarted, gets recovered too). */
+    recoveredPids = new Set();
+    recovering;
+    /** Wire v5: starts in flight by keeper exec id (a repeated request joins them), exec_cancel received meanwhile. */
+    startingDurable = new Map();
+    pendingCancels = new Map();
+    /** Restores in flight by keeper exec id (recovery and `find` share them: never two followers for one exec). */
+    loading = new Map();
+    frameTooLargeLogged = false;
+    /** exec_ack for execs not followed yet (by keeper exec id, highest offset), applied on adoption. Insertion-ordered, bounded. */
+    pendingAcks = new Map();
+    /** exec_cancel for execs `find` could not decide about (by keeper exec id): applied when recovery adopts them. Bounded. */
+    cancelOnAdopt = new Map();
+    /** Wire v5 fallback execs this daemon ran, by keeper exec id (insertion-ordered, bounded): see {@link FallbackExecRecord}. */
+    fallbackExecs = new Map();
+    /**
+     * Keeper exec ids whose start may have reached keeper-terminal, or that a server asked to re-attach (insertion-ordered, bounded).
+     * A request for one of them is a REPEAT: if keeper-terminal cannot be reached right now it may still be running there, so a repeat
+     * never falls back to this process (Task 26) — the answer is "unconfirmed" instead of a possible second run.
+     */
+    seenIds = new Set();
+    constructor(options = {}) {
+        this.options = options;
+        this.now = options.now ?? Date.now;
+        this.log = options.log ?? (() => { });
+        this.limit = options.limit ?? EXEC_CONCURRENCY_LIMIT;
+        this.killGraceMs = options.killGraceMs ?? EXEC_KILL_GRACE_MS;
+    }
+    /** Commands running now: in a run(), plus durable ones followed without one (detached, recovered and still being stopped). */
+    get activeCount() {
+        let followed = 0;
+        for (const exec of this.followed.values())
+            if (!exec.finished)
+                followed += 1;
+        return this.running + followed;
+    }
+    /** A durable exec followed outside a run (recovered or detached), for the server to re-attach (wire v5, Task 12). */
+    followedExec(execId) { return this.followed.get(execId); }
+    /** Where a new exec starts; null = no keeper-terminal at all. */
+    async servingKeeper() {
+        const generations = this.options.generations;
+        if (generations)
+            return generations.serving();
+        return this.options.keeper ?? null;
+    }
+    /**
+     * Every keeper-terminal generation this daemon can route to, the serving one first (none is started here). Rejects when the other
+     * generations cannot be listed: "no older generation" is never assumed (an older one may run the exec asked about).
+     */
+    async allKeepers() {
+        const generations = this.options.generations;
+        if (!generations)
+            return this.options.keeper ? [this.options.keeper] : [];
+        const [serving, others] = await Promise.all([generations.serving(), generations.others()]);
+        return [serving, ...others.filter((keeper) => keeper !== serving)];
+    }
+    /**
+     * The hello of `keeper` when it runs; null only when it is provably absent (or, for a keeper without `probe`, when `runningHello`
+     * says so). Throws {@link ExecStateUndecidableError} when it could not be told.
+     */
+    async presentHello(keeper) {
+        if (!keeper.probe)
+            return keeper.runningHello();
+        const presence = await keeper.probe(this.options.probeTimeoutMs ?? EXEC_PROBE_TIMEOUT_MS);
+        if (presence.state === 'running')
+            return presence.hello;
+        if (presence.state === 'absent')
+            return null;
+        throw new ExecStateUndecidableError(`keeper-terminal could not be reached: ${presence.reason}`);
+    }
+    markSeen(execId) {
+        this.seenIds.delete(execId);
+        this.seenIds.add(execId);
+        if (this.seenIds.size > SEEN_IDS_LIMIT)
+            this.seenIds.delete(this.seenIds.values().next().value);
+    }
+    durableOptions() {
+        return { ...this.options.durable, now: this.now, killGraceMs: this.killGraceMs, log: this.log };
+    }
+    /** `ComputerExecutor`: runs one `exec_request` and answers its `exec_exit`. Never rejects. */
+    async run(req, onOutput, signal, hooks = {}) {
+        if (this.activeCount >= this.limit)
+            return exitFrame(req.id, EXEC_LIMIT_MESSAGE);
+        this.running += 1;
+        try {
+            return await this.execute(req, onOutput, signal, hooks);
+        }
+        catch (error) {
+            return exitFrame(req.id, messageOf(error));
+        }
+        finally {
+            this.running -= 1;
+        }
+    }
+    async execute(req, onOutput, signal, hooks) {
+        const receivedAtMs = this.now();
+        const deadlineEpochMs = execDeadline(req, receivedAtMs);
+        const env = execEnvironment(this.options.env?.() ?? process.env, req);
+        if (signal?.aborted)
+            return exitFrame(req.id, '');
+        const execId = this.options.newExecId?.() ?? `${hooks.outlivesChannel ? '' : CHANNEL_BOUND_EXEC_PREFIX}${randomUUID()}`;
+        const starting = this.start(req, env, deadlineEpochMs, { execId, outlivesChannel: hooks.outlivesChannel === true });
+        // exec_cancel, a channel close or a force stop answers at once, however long keeper-terminal takes to answer the start.
+        const started = await untilAborted(starting, signal);
+        if (started === ABORTED) {
+            void starting.then((late) => {
+                if (late.kind !== 'durable')
+                    return;
+                this.stopUnowned(late.exec); // it started after all: stop it like the abort would have
+            }, (error) => this.log(`exec ${req.id}: ${messageOf(error)}`));
+            return exitFrame(req.id, '');
+        }
+        if (started.kind === 'exit')
+            return started.exit;
+        if (started.kind === 'fallback') {
+            if (this.options.keeper || this.options.generations)
+                this.log(`exec ${req.id}: running in the daemon (not durable): ${started.why}`);
+            return this.runInProcess(req, env, deadlineEpochMs, onOutput, signal, hooks);
+        }
+        return this.follow(req, started.exec, onOutput, signal, hooks);
+    }
+    /** Answer timeout of exec.start / exec.status: a few seconds, never longer than the time left before the command's deadline. */
+    callTimeout(deadlineEpochMs) {
+        const base = this.options.startTimeoutMs ?? EXEC_START_TIMEOUT_MS;
+        return deadlineEpochMs === undefined ? base : Math.max(MIN_CALL_TIMEOUT_MS, Math.min(base, deadlineEpochMs - this.now()));
+    }
+    /**
+     * keeper-terminal when it is reachable and serves exec. The fallback only when the keeper certainly did not start the command:
+     * the request never left (launch/connect failure, not connected, frame too large, protocol window) or the keeper refused it.
+     * Any other failure (connection lost or closed after the write, timeout) is settled by `exec.status`; an `exec_not_found` counts
+     * only from the very keeper process that received the start. Undecidable: an error result, never a second run, and the exec id is
+     * left to recovery (which stops it if it did start).
+     */
+    async start(req, env, deadlineEpochMs, how) {
+        const { execId, outlivesChannel, wire } = how;
+        const keeper = await this.servingKeeper();
+        if (!keeper)
+            return { kind: 'fallback', why: 'no keeper-terminal' };
+        let hello;
+        try {
+            hello = (await keeper.ensure()).hello;
+        }
+        catch (error) {
+            // A repeat may be running in a keeper-terminal this daemon cannot reach right now: running it here too could run it twice.
+            if (how.repeat) {
+                this.log(`exec ${req.id}: repeated request while keeper-terminal is unreachable: ${messageOf(error)}`);
+                return { kind: 'exit', exit: exitFrame(req.id, EXEC_UNCONFIRMED_MESSAGE) };
+            }
+            return { kind: 'fallback', why: `keeper-terminal unavailable: ${messageOf(error)}` };
+        }
+        if (!hello?.capabilities.includes(EXEC_CAPABILITY))
+            return { kind: 'fallback', why: 'keeper-terminal does not serve exec' };
+        if (!this.recoveredPids.has(hello.pid))
+            void this.recover(); // a keeper process this daemon has not recovered yet
+        const params = {
+            execId, argv: execArgv(req), cwd: execCwd(req), env,
+            ...(deadlineEpochMs !== undefined ? { deadlineEpochMs } : {}),
+            killGraceMs: this.killGraceMs, retainHeadBytes: EXEC_RETAIN_HEAD_BYTES, maxUncommittedBytes: EXEC_MAX_UNCOMMITTED_BYTES,
+            onKeeperLoss: 'kill', label: how.label || EXEC_DEFAULT_LABEL,
+            // keeper-terminal refuses a jobId beyond its bound; it only labels activity items, so an oversized one is dropped, not fatal.
+            ...(how.jobId && how.jobId.length <= EXEC_JOB_ID_CHARS ? { jobId: how.jobId } : {}),
+        };
+        this.starting.add(execId); // until follow() (or a late stop) owns it: recovery must never take it
+        let owned = false;
+        try {
+            const before = await how.beforeStart?.(keeper);
+            if (before) {
+                owned = before.kind === 'durable';
+                return before;
+            }
+            if (wire)
+                this.markSeen(execId); // from here on it may exist in keeper-terminal
+            const descriptor = await keeper.execStart(params, { timeoutMs: this.callTimeout(deadlineEpochMs) });
+            owned = true;
+            return { kind: 'durable', exec: DurableExec.follow(keeper, descriptor, { outlivesChannel, ...(wire ? { wire } : {}) }, this.durableOptions()) };
+        }
+        catch (error) {
+            // Another keeper-terminal generation owns an exec of this id (spec §3.1: two generations may coexist): it exists, so it is
+            // never started again here and never run in this process.
+            if (errorCode(error) === ExecErrorCode.foreign) {
+                this.log(`exec ${req.id}: owned by another keeper-terminal generation`);
+                return { kind: 'exit', exit: exitFrame(req.id, EXEC_UNCONFIRMED_MESSAGE) };
+            }
+            if (startedNothing(error)) {
+                if (error instanceof KeeperFrameTooLargeError && !this.frameTooLargeLogged) {
+                    this.frameTooLargeLogged = true; // once per daemon: every such command runs in the daemon, not durable
+                    this.log(`exec.start frames over keeper-terminal's inbound cap (a very large command or environment) run in the daemon, not durable`);
+                }
+                return { kind: 'fallback', why: `exec.start refused: ${messageOf(error)}` };
+            }
+            try {
+                const status = await keeper.execStatus(execId, { timeoutMs: this.callTimeout(deadlineEpochMs) });
+                const exec = await DurableExec.restore(keeper, status, { ...this.durableOptions(), outlivesChannel, ...(wire ? { wire } : {}) });
+                owned = true;
+                return { kind: 'durable', exec };
+            }
+            catch (probe) {
+                if (errorCode(probe) === 'exec_not_found') {
+                    const now = await keeper.runningHello().catch(() => null);
+                    if (now && now.pid === hello.pid && now.gen === hello.gen)
+                        return { kind: 'fallback', why: `exec.start failed: ${messageOf(error)}` };
+                }
+                this.log(`exec ${req.id}: start unconfirmed: ${messageOf(error)}; ${messageOf(probe)}`);
+                this.recoverLater();
+                return { kind: 'exit', exit: exitFrame(req.id, EXEC_UNCONFIRMED_MESSAGE) };
+            }
+        }
+        finally {
+            if (!owned)
+                this.starting.delete(execId);
+        }
+    }
+    /**
+     * An exec this daemon no longer follows may still run in keeper-terminal (an unconfirmed start, a keeper that stopped
+     * answering): the next time the daemon reaches the keeper it runs recovery again, which stops and deletes every exec nobody owns.
+     */
+    recoverLater() { this.recoveredPids.clear(); }
+    async follow(req, exec, onOutput, signal, hooks) {
+        this.inRun.add(exec.execId);
+        this.starting.delete(exec.execId);
+        hooks.onStarted?.({ durable: true, execId: exec.execId, ...(exec.pid !== null ? { pid: exec.pid } : {}), startedAtMs: this.now() });
+        exec.attach(req.streaming && onOutput ? (stream, text) => onOutput(stream, text) : undefined);
+        const onAbort = () => { void exec.cancel(); };
+        if (signal?.aborted)
+            onAbort();
+        else
+            signal?.addEventListener('abort', onAbort, { once: true });
+        try {
+            const outcome = await exec.outcome;
+            if (outcome.state === 'unknown')
+                this.recoverLater();
+            // Wire ≤ 4: the result goes out first; the keeper deletes the exec afterwards (wire v5 waits for the server's ack instead).
+            setImmediate(() => { void exec.release(); });
+            return durableExitFrame(req.id, outcome);
+        }
+        finally {
+            signal?.removeEventListener('abort', onAbort);
+            this.inRun.delete(exec.execId);
+        }
+    }
+    runInProcess(req, env, deadlineEpochMs, onOutput, signal, hooks) {
+        // Never spawn what is already cancelled or already past its deadline.
+        if (signal?.aborted)
+            return Promise.resolve(exitFrame(req.id, ''));
+        if (deadlineEpochMs !== undefined && deadlineEpochMs <= this.now())
+            return Promise.resolve({ ...exitFrame(req.id, ''), timedOut: true });
+        const running = startInProcess(req, onOutput, { env, deadlineEpochMs, now: this.now });
+        hooks.onStarted?.({ durable: false, ...(running.pid !== undefined ? { pid: running.pid } : {}), startedAtMs: this.now() });
+        const onAbort = () => { void running.cancel(); }; // an in-process exec dies with its channel, whatever the wire
+        if (signal?.aborted)
+            onAbort();
+        else
+            signal?.addEventListener('abort', onAbort, { once: true });
+        return running.result.finally(() => signal?.removeEventListener('abort', onAbort));
+    }
+    /** Follows `exec` outside any run only to stop it (SIGTERM, grace, SIGKILL; or completing a cancel already begun) and delete it. */
+    stopUnowned(exec) {
+        const { execId } = exec;
+        this.starting.delete(execId);
+        this.followed.set(execId, exec);
+        void exec.cancel();
+        void exec.outcome.then((outcome) => {
+            if (outcome.state === 'unknown')
+                this.recoverLater();
+            return exec.release();
+        }).finally(() => { if (this.followed.get(execId) === exec)
+            this.followed.delete(execId); });
+    }
+    /**
+     * Daemon start (and whenever the daemon reaches a keeper-terminal process it has not recovered, or after it lost track of an
+     * exec): follows every exec keeper-terminal still has that nobody here owns. Each is restored from its checkpoint. One that does
+     * not outlive its channel (all of them under wire ≤ 4) lost its channel and the server already reported it ended: it is stopped
+     * (SIGTERM, grace, SIGKILL, also completing a cancel the previous daemon began) and deleted once ended. One that outlives its
+     * channel stays followed for the server to re-attach. Starts no keeper. One recovery at a time (callers join the one in flight).
+     */
+    recover() {
+        this.recovering ??= this.recoverOnce().finally(() => { this.recovering = undefined; });
+        return this.recovering;
+    }
+    async recoverOnce() {
+        let keepers;
+        try {
+            keepers = await this.allKeepers();
+        }
+        catch (error) {
+            this.log(`exec recovery: ${messageOf(error)}`);
+            this.recoverLater();
+            return;
+        }
+        // Every running generation: an older one keeps the execs it runs (they are followed through its own client, spec §8.3).
+        for (const keeper of keepers)
+            await this.recoverFrom(keeper);
+    }
+    async recoverFrom(keeper) {
+        let ids;
+        let pid;
+        try {
+            // Absent (proven): nothing to recover there. Not answering: thrown, so recovery runs again later (a slow keeper is not a gone one).
+            const hello = await this.presentHello(keeper);
+            if (!hello?.capabilities.includes(EXEC_CAPABILITY))
+                return;
+            pid = hello.pid;
+            ids = (await keeper.execList()).execs.map((exec) => exec.execId);
+        }
+        catch (error) {
+            this.log(`exec recovery: ${messageOf(error)}`);
+            this.recoverLater();
+            return;
+        }
+        // Marked only once the list is in hand: a failed list leaves this keeper process to the next start's recovery.
+        this.recoveredPids.add(pid);
+        for (const execId of ids) {
+            if (this.owned(execId))
+                continue;
+            try {
+                await this.load(execId, keeper);
+            }
+            catch (error) {
+                this.log(`exec recovery ${execId}: ${messageOf(error)}`);
+                this.recoverLater();
+            }
+        }
+    }
+    /** Followed, being started or in a run: somebody here owns it. */
+    owned(execId) {
+        return this.starting.has(execId) || this.inRun.has(execId) || this.followed.has(execId) || this.startingDurable.has(execId);
+    }
+    /**
+     * Restores `execId` from keeper-terminal and follows it (recovery and `find` share this; one restore per exec at a time): one
+     * that outlives its channel (wire v5) for the server to re-attach, any other (its channel died with the daemon that started it,
+     * and the server already reported it ended) only to stop it (SIGTERM, grace, SIGKILL, also completing a cancel the previous daemon
+     * began) and delete it once ended. null: the keeper does not know it. Throws when the keeper could not tell. Starts no keeper.
+     */
+    load(execId, keeper) {
+        const inFlight = this.loading.get(execId);
+        if (inFlight)
+            return inFlight;
+        const loading = (async () => {
+            let status;
+            try {
+                status = await keeper.execStatus(execId);
+            }
+            catch (error) {
+                if (errorCode(error) === ExecErrorCode.notFound)
+                    return null;
+                throw error;
+            }
+            // Without a checkpoint (the daemon died between exec.start and the first one) the id tells: a channel-bound wire ≤ 4 exec is
+            // stopped, anything else is a wire v5 exec the server may still re-attach (conservatively non-streaming: it gets the exit).
+            const exec = await DurableExec.restore(keeper, status, {
+                ...this.durableOptions(), outlivesChannel: !channelBound(execId), ...(channelBound(execId) ? {} : { wire: { streaming: false } }),
+            });
+            const owner = this.followed.get(execId);
+            if (owner || this.starting.has(execId) || this.inRun.has(execId) || this.startingDurable.has(execId)) {
+                exec.abandon();
+                return owner ?? null;
+            }
+            if (exec.outlivesChannel)
+                this.adopt(execId, exec);
+            else
+                this.stopUnowned(exec);
+            return exec;
+        })().finally(() => { this.loading.delete(execId); });
+        this.loading.set(execId, loading);
+        return loading;
+    }
+    /**
+     * Wire v5: this policy follows `exec` for the server (no channel owns it). Once ended and not released by the server's ack it is
+     * forgotten after keeper-terminal's own log TTL (keeper-terminal deletes it then anyway).
+     */
+    adopt(execId, exec) {
+        this.followed.set(execId, exec);
+        const cancel = this.cancelOnAdopt.get(execId);
+        if (cancel) {
+            this.cancelOnAdopt.delete(execId);
+            void exec.cancel(cancel);
+        }
+        const acked = this.pendingAcks.get(execId);
+        if (acked !== undefined) {
+            this.pendingAcks.delete(execId);
+            this.applyAck(execId, exec, acked);
+            // A restored exec learns its final head only once it has processed its exit record: the ack may be the final one.
+            if (!exec.finished)
+                void exec.outcome.then(() => { if (this.followed.get(execId) === exec)
+                    this.applyAck(execId, exec, acked); });
+        }
+        void exec.outcome.then((outcome) => {
+            if (outcome.state === 'unknown')
+                this.recoverLater();
+            const timer = setTimeout(() => { if (this.followed.get(execId) === exec)
+                this.followed.delete(execId); }, EXEC_TERMINAL_TTL_MS);
+            timer.unref();
+        });
+    }
+    // ---- COMPUTER wire v5 (spec §4.3): execs the server can re-attach to ------------------------------------------------
+    /**
+     * An `exec_request` under wire v5. The keeper exec id is the server's id ({@link keeperExecIdOf}): an id this daemon or
+     * keeper-terminal already knows is that exec (attach; never a second run). Otherwise it starts in keeper-terminal (durable) when
+     * it can; the fallback is left to the caller (`run`), which decides whether to admit it (spec §8.2: refused while draining).
+     * Never rejects.
+     */
+    startOrFind(req) {
+        const receivedAtMs = this.now();
+        const execId = keeperExecIdOf(req.id);
+        // Synchronous up to the registration below: two requests for one id can never both start it.
+        const inFlight = this.startingDurable.get(execId);
+        if (inFlight)
+            return inFlight.then((first) => this.repeated(req, first));
+        const followed = this.followed.get(execId);
+        if (followed)
+            return Promise.resolve({ kind: 'durable', exec: followed, existed: true });
+        // It ran (or runs) in this process: never again. Its exit answers the repeat once it ended (a closed channel already stopped it).
+        const fallback = this.fallbackOf(req.id);
+        if (fallback)
+            return fallback.exit.then((exit) => ({ kind: 'exit', exit: { ...exit, id: req.id } }));
+        const repeat = this.seenIds.has(execId);
+        const starting = (async () => {
+            const loaded = await this.loading.get(execId)?.catch(() => null); // recovery is restoring it right now
+            if (loaded)
+                return { kind: 'durable', exec: loaded, existed: true };
+            if (this.activeCount >= this.limit)
+                return { kind: 'exit', exit: exitFrame(req.id, EXEC_LIMIT_MESSAGE) };
+            const deadlineEpochMs = execDeadline(req, receivedAtMs);
+            const env = execEnvironment(this.options.env?.() ?? process.env, req);
+            return await this.startDurable(req, execId, env, deadlineEpochMs, repeat);
+        })().catch((error) => ({ kind: 'exit', exit: exitFrame(req.id, messageOf(error)) }));
+        this.startingDurable.set(execId, starting);
+        void starting.finally(() => { if (this.startingDurable.get(execId) === starting)
+            this.startingDurable.delete(execId); });
+        return starting;
+    }
+    /** A request repeated while the first was being started: the same durable exec, never a second run of anything. */
+    repeated(req, first) {
+        if (first.kind === 'durable')
+            return { ...first, existed: true };
+        if (first.kind === 'exit')
+            return { kind: 'exit', exit: { ...first.exit, id: req.id } };
+        return { kind: 'exit', exit: exitFrame(req.id, 'The same command is already running in the daemon for another request') };
+    }
+    /** An exec this daemon follows or is starting under `execId`. */
+    async known(execId) {
+        const followed = this.followed.get(execId);
+        if (followed)
+            return followed;
+        const starting = this.startingDurable.get(execId);
+        if (starting) {
+            const started = await starting;
+            return started.kind === 'durable' ? started.exec : null;
+        }
+        const loading = this.loading.get(execId);
+        if (loading)
+            return loading.catch(() => null);
+        return null;
+    }
+    async startDurable(req, execId, env, deadlineEpochMs, repeat) {
+        this.running += 1;
+        let started;
+        let existed = false;
+        try {
+            started = await this.start(req, env, deadlineEpochMs, {
+                execId, outlivesChannel: true, wire: { streaming: req.streaming }, label: req.label, jobId: req.jobId, repeat,
+                // Already in keeper-terminal (a repeated request after a daemon restart, before recovery reached it) — in the serving
+                // generation or an older one still running (routed by owner, Task 26): that exec.
+                beforeStart: async (serving) => {
+                    // Other generations are asked best-effort (final review finding 1, correction): one that cannot be listed or reached
+                    // (slow, hung, outside the protocol window) is skipped, never a reason to refuse every new exec — keeper-terminal's own
+                    // `exec.start` refuses an id another live generation holds (`exec_foreign`, exec dirs are shared), which `start` answers as
+                    // unconfirmed. `find` and recovery keep the strict rule.
+                    let others;
+                    try {
+                        others = (await this.allKeepers()).filter((keeper) => keeper !== serving);
+                    }
+                    catch (error) {
+                        this.log(`exec ${req.id}: could not list keeper-terminal generations (relying on keeper-terminal's foreign guard): ${messageOf(error)}`);
+                        others = [];
+                    }
+                    for (const keeper of [serving, ...others]) {
+                        try {
+                            if (keeper !== serving && !(await this.presentHello(keeper))?.capabilities.includes(EXEC_CAPABILITY))
+                                continue;
+                            const status = await keeper.execStatus(execId, { timeoutMs: this.callTimeout(deadlineEpochMs) });
+                            const exec = await DurableExec.restore(keeper, status, { ...this.durableOptions(), outlivesChannel: true, wire: { streaming: req.streaming } });
+                            existed = true;
+                            return { kind: 'durable', exec };
+                        }
+                        catch (error) {
+                            // Only an explicit "not found" proves it does not run there.
+                            if (errorCode(error) === ExecErrorCode.notFound)
+                                continue;
+                            if (keeper !== serving) {
+                                this.log(`exec ${req.id}: an older keeper-terminal generation could not tell (skipped): ${messageOf(error)}`);
+                                continue;
+                            }
+                            // The serving generation could not tell (not connected, refused, timed out): no start, and certainly no in-process copy.
+                            this.log(`exec ${req.id}: could not tell whether it already runs: ${messageOf(error)}`);
+                            return { kind: 'exit', exit: exitFrame(req.id, EXEC_UNCONFIRMED_MESSAGE) };
+                        }
+                    }
+                    return null;
+                },
+            });
+        }
+        finally {
+            this.running -= 1;
+        }
+        if (started.kind !== 'durable')
+            this.pendingCancels.delete(execId);
+        if (started.kind === 'exit')
+            return started;
+        if (started.kind === 'fallback') {
+            if (this.options.keeper || this.options.generations)
+                this.log(`exec ${req.id}: running in the daemon (not durable): ${started.why}`);
+            return {
+                kind: 'fallback', why: started.why,
+                run: (onOutput, signal, onStarted) => this.runFallback(req, env, deadlineEpochMs, onOutput, signal, onStarted),
+            };
+        }
+        const { exec } = started;
+        this.starting.delete(execId);
+        this.adopt(execId, exec);
+        const cancel = this.pendingCancels.get(execId);
+        if (cancel) {
+            this.pendingCancels.delete(execId);
+            void exec.cancel(cancel);
+        }
+        return { kind: 'durable', exec, existed };
+    }
+    /**
+     * The in-process fallback of a v5 request, counted against the concurrency limit while it runs. Remembered once it started
+     * ({@link fallbackOf}), so another channel's re-attach or repeated request learns it ran here instead of starting it again.
+     */
+    async runFallback(req, env, deadlineEpochMs, onOutput, signal, onStarted) {
+        if (this.activeCount >= this.limit)
+            return exitFrame(req.id, EXEC_LIMIT_MESSAGE);
+        const execId = keeperExecIdOf(req.id);
+        let settle;
+        const record = { done: false, endedAt: 0, exit: new Promise((resolve) => { settle = resolve; }) };
+        let remembered = false;
+        const remember = () => {
+            if (remembered)
+                return;
+            remembered = true;
+            this.fallbackExecs.delete(execId);
+            this.fallbackExecs.set(execId, record);
+            this.trimFallbacks();
+        };
+        this.running += 1;
+        let exit;
+        try {
+            exit = await this.runInProcess(req, env, deadlineEpochMs, onOutput, signal, { onStarted: (info) => { remember(); onStarted(info); } });
+        }
+        catch (error) {
+            exit = exitFrame(req.id, messageOf(error));
+        }
+        finally {
+            this.running -= 1;
+        }
+        record.done = true;
+        record.endedAt = this.now();
+        settle({ ...exit, cancelled: signal.aborted });
+        this.trimFallbacks();
+        return exit;
+    }
+    /**
+     * Bounds {@link fallbackExecs}: ended records past {@link FALLBACK_RECORD_TTL_MS}, and the oldest ended ones beyond
+     * {@link FALLBACK_OUTPUT_RECORDS_LIMIT}, drop their output (tombstones); then the oldest ended records go, then the oldest of all.
+     */
+    trimFallbacks() {
+        const ended = [...this.fallbackExecs].filter(([, record]) => record.done && !record.tombstone);
+        const now = this.now();
+        ended.forEach(([id, record], index) => {
+            if (now - record.endedAt <= FALLBACK_RECORD_TTL_MS && ended.length - index <= FALLBACK_OUTPUT_RECORDS_LIMIT)
+                return;
+            const tombstone = {
+                done: true, endedAt: record.endedAt, tombstone: true,
+                exit: record.exit.then((exit) => ({ t: 'exec_exit', id: exit.id, exitCode: null, stdout: '', stderr: FALLBACK_EXPIRED_NOTE, timedOut: false, truncated: false, cancelled: exit.cancelled })),
+            };
+            this.fallbackExecs.set(id, tombstone); // same position: insertion order is kept
+        });
+        while (this.fallbackExecs.size > FALLBACK_RECORDS_LIMIT) {
+            let victim;
+            for (const [id, record] of this.fallbackExecs)
+                if (record.done) {
+                    victim = id;
+                    break;
+                }
+            this.fallbackExecs.delete(victim ?? this.fallbackExecs.keys().next().value);
+        }
+    }
+    /** See {@link DurableExecHost.fallbackOf}. */
+    fallbackOf(serverId) {
+        this.trimFallbacks();
+        return this.fallbackExecs.get(keeperExecIdOf(serverId));
+    }
+    /**
+     * `exec_attach` / `exec_status`: the exec with the server's id `serverId`, followed here or restored from keeper-terminal (after a
+     * daemon restart, before recovery reached it). null = unknown, and only when that is proven: every keeper-terminal generation was
+     * reached and none knows it (never started, deleted by the TTL or released), or none runs at all. Never starts a keeper. A keeper
+     * that cannot answer (slow, not responding, a listing that failed) is asked again a few times; still undecided, this throws
+     * {@link ExecStateUndecidableError} — never null, which would make the server give the exec up while it may be running.
+     */
+    async find(serverId) {
+        const execId = keeperExecIdOf(serverId);
+        // A server asks about it: whatever this answers, a later request for the same id is a repeat (see `seenIds`).
+        this.markSeen(execId);
+        const known = await this.known(execId);
+        if (known)
+            return known;
+        for (let attempt = 1;; attempt += 1) {
+            try {
+                // Every running generation, by owner (Task 26): an exec an older keeper-terminal runs is found there.
+                for (const keeper of await this.allKeepers()) {
+                    const hello = await this.presentHello(keeper);
+                    if (!hello?.capabilities.includes(EXEC_CAPABILITY))
+                        continue; // provably absent, or serves no exec: holds none
+                    const exec = await this.load(execId, keeper);
+                    if (exec)
+                        return exec;
+                }
+                return null;
+            }
+            catch (error) {
+                if (attempt >= (this.options.findAttempts ?? FIND_ATTEMPTS)) {
+                    this.log(`exec ${serverId}: state undecided: ${messageOf(error)}`);
+                    this.recoverLater();
+                    throw new ExecStateUndecidableError(`Could not tell whether the command still runs: ${messageOf(error)}`);
+                }
+                await new Promise((resolve) => setTimeout(resolve, FIND_RETRY_MS));
+            }
+        }
+    }
+    /**
+     * `exec_cancel`: SIGTERM (or the given signal), SIGKILL after the grace. Idempotent; silent for an unknown or ended exec. A cancel
+     * that arrives while the exec is still being started applies as soon as it runs.
+     */
+    async cancel(serverId, signal) {
+        const execId = keeperExecIdOf(serverId);
+        const first = isExecSignal(signal) ? signal : 'SIGTERM';
+        const starting = this.startingDurable.get(execId);
+        if (starting) {
+            this.pendingCancels.set(execId, first); // applied the moment the start lands …
+            const started = await starting;
+            this.pendingCancels.delete(execId);
+            if (started.kind === 'durable')
+                await started.exec.cancel(first); // … or here (idempotent)
+            return;
+        }
+        let exec;
+        try {
+            exec = await this.find(serverId);
+        }
+        catch (error) {
+            // Undecided: the cancel is kept and applied the moment recovery adopts the exec (never dropped while it may be running).
+            this.cancelOnAdopt.delete(execId);
+            this.cancelOnAdopt.set(execId, first);
+            if (this.cancelOnAdopt.size > PENDING_ACKS_LIMIT)
+                this.cancelOnAdopt.delete(this.cancelOnAdopt.keys().next().value);
+            throw error;
+        }
+        await exec?.cancel(first);
+    }
+    /**
+     * `exec_ack`: the server persisted the output through `throughOffset`. Commits follow (exec/durable.ts); an ack that reaches the
+     * final head of an ended exec (the `exec_status` sent before its `exec_exit`) releases it: keeper-terminal deletes it.
+     */
+    acknowledge(serverId, throughOffset) {
+        const execId = keeperExecIdOf(serverId);
+        const exec = this.followed.get(execId);
+        if (exec) {
+            this.applyAck(execId, exec, throughOffset);
+            return;
+        }
+        // Not followed (yet): a restarted daemon whose recovery has not reached it. Kept (bounded) and applied when it is adopted.
+        if (throughOffset <= (this.pendingAcks.get(execId) ?? -2))
+            return;
+        this.pendingAcks.delete(execId);
+        this.pendingAcks.set(execId, throughOffset);
+        if (this.pendingAcks.size > PENDING_ACKS_LIMIT)
+            this.pendingAcks.delete(this.pendingAcks.keys().next().value);
+    }
+    applyAck(execId, exec, throughOffset) {
+        exec.acknowledge(throughOffset);
+        if (exec.finished && throughOffset >= exec.finalHead) {
+            if (this.followed.get(execId) === exec)
+                this.followed.delete(execId);
+            void exec.release();
+        }
+    }
+}