@dynamicagents/plugins 0.12.0 → 0.14.0

package/dist/{computer/host → workspace}/install-job.js

@@ -1,14 +1,14 @@
 import { shellQuote } from "@cloudflare/computer";
 import { JobLifecycle } from "@dynamicagents/core/job";
-// Leaf modules rather than `../index.js`, and that is structural: the barrel
+// Leaf modules rather than `./index.ts`, and that is structural: the barrel
 // re-exports this directory, so reaching it from here would be a cycle — through
 // a module that builds a class at import time, where initialisation order
 // decides whether the base is `undefined`.
-import { installFingerprint, resolveInstallCommand } from "../install.js";
-import { pathExists } from "../read.js";
+import { installFingerprint, resolveInstallCommand, rootFingerprint } from "./install.js";
+import { pathExists } from "./read.js";
 import { deploymentFault } from "./container-fault.js";
 import { INSTALLED_MARKER } from "./container-deps.js";
-import { truncateOutput } from "../render.js";
+import { truncateOutput } from "./format.js";
 /**
  * Whether a thrown value is the runtime reporting a replaced container.
  *
@@ -27,7 +27,7 @@ function execWasLost(err) {
  * ## The rule every guard here serves
  *
  * `running` is the one install state that **blocks work**: the plugin's gate
- * waits on it and then refuses to run. Every other state is a fact a subagent
+ * waits on it and then refuses to run. Every other state is a fact an agent
  * can act on. So a `running` record must never outlive the command it describes,
  * and the ways it can are not all reachable from one place — the spawn can fail
  * before a drain is attached, a drain can be cut short by an eviction, `getExec`
@@ -38,8 +38,9 @@ function execWasLost(err) {
  * ## What decides that an install is needed
  *
  * The tree lives on the container's disk (see `./container-deps.ts`), so it goes
- * with the container. {@link CONTAINER_TREE_KEY} records a successful install and
- * is cleared whenever that container is seen gone.
+ * with the container. {@link CONTAINER_TREE_KEY} records a successful install — or
+ * a tree restored with a snapshot of the container that ran one, through
+ * {@link InstallJob.adopt} — and is cleared whenever that container is seen gone.
  */
 /**
  * The install's job id, which is also the key its state record lives under.
@@ -78,6 +79,11 @@ const INSTALL_STALE_MS = 5 * 60_000;
 const CONTAINER_TREE_KEY = "install:tree";
 /** Records a deployed object may still hold, deleted on the next install. */
 const RETIRED_KEYS = ["install:syncing", "install:completed"];
+/**
+ * The command a reservation's placeholder names, until the install's own is
+ * resolved. The gate renders it while it waits.
+ */
+const RESOLVING = "(being resolved)";
 /**
  * The exec id an install runs under.
  *
@@ -108,8 +114,12 @@ export class InstallJob {
      *
      * **Arming writes `running` before anything is running**, which holds the gate
      * shut in the moments before the alarm fires. The alarm then presents the stamp
-     * arming wrote to `claim`, which recognises its own placeholder and nothing
+     * arming wrote to `reserve`, which recognises its own placeholder and nothing
      * else — taking over any `running` record instead would be displacement again.
+     *
+     * Every write a run makes goes through the lifecycle's `generation`, under
+     * the `startedAt` its reservation returned, so a run that has been displaced
+     * writes nothing.
      */
     #job;
     // Built here rather than as a field initializer, which would run before the
@@ -181,7 +191,7 @@ export class InstallJob {
             dir: context.dir
         });
         // Everything the arming handshake needs — the placeholder write, the stamp
-        // the alarm presents to `claim`, the cooldown floor and the run intent — in
+        // the alarm presents to `reserve`, the cooldown floor and the run intent — in
         // one call, and unwound as a unit if the intent cannot be scheduled.
         await this.#job.arm({ command: state.command });
     }
@@ -203,6 +213,57 @@ export class InstallJob {
     async dir() {
         return (await this.#job.context())?.dir;
     }
+    /** Where the last install ran, and for which repository's override. */
+    async target() {
+        const context = await this.#job.context();
+        if (!context?.dir)
+            return undefined;
+        return {
+            dir: context.dir,
+            ...(context.repo ? { repo: context.repo } : {})
+        };
+    }
+    /**
+     * The fingerprint an install into `target` would write as its marker, from the
+     * checkout as it stands — `null` when there is nothing to install there.
+     */
+    async fingerprintFor(target) {
+        const probe = this.#probe();
+        const resolution = await resolveInstallCommand(probe, target.dir, this.deps.plan(), target.repo);
+        return await installFingerprint(probe, target.dir, resolution);
+    }
+    /** What `dir`'s install inputs hash to — see `rootFingerprint`. */
+    async rootFingerprint(dir) {
+        return await rootFingerprint(this.#probe(), dir, this.deps.plan());
+    }
+    /** The tree this container holds, if a finished install says it does. */
+    async tree() {
+        const tree = await this.#tree();
+        return (tree && { dir: tree.dir, fingerprint: tree.fingerprint, at: tree.at });
+    }
+    /**
+     * Record a tree this container came up holding — restored with a snapshot of
+     * the container that installed it — as if this container had installed it.
+     *
+     * Only for a tree whose marker the setup read and matched against the checkout
+     * as it stands; see `./deps-snapshot.ts`. So the last install's context moves
+     * to it too: a checkout that moved since that install would otherwise read as
+     * missing its tree, and arm an install that finds the tree already there.
+     */
+    async adopt(tree) {
+        await this.deps.storage.put(CONTAINER_TREE_KEY, {
+            dir: tree.dir,
+            fingerprint: tree.fingerprint,
+            at: Date.now()
+        });
+        const context = await this.#job.context();
+        if (context?.dir === tree.dir && context.fingerprint !== tree.fingerprint)
+            await this.#job.putContext({ ...context, fingerprint: tree.fingerprint });
+        console.info(`[${this.deps.tag()}] adopted a restored dependency tree`, {
+            id: this.deps.id(),
+            dir: tree.dir
+        });
+    }
     /** The container went away, and its tree with it. */
     async containerGone() {
         await this.deps.storage.delete(CONTAINER_TREE_KEY);
@@ -239,7 +300,7 @@ export class InstallJob {
      *
      * Called from `repo_clone` through the repo plugin's `afterCheckout` hook, so
      * it runs inside a model turn and must not block on the install — 225 seconds
-     * for slack-gatekeeper, against a chunk step that dies at ten minutes.
+     * for slack-gatekeeper, against a turn that is cut at fifteen minutes.
      *
      * That caller is a model turn, which lives long enough to hold the drain handed
      * to `ctx.waitUntil` below. **A short-lived caller cannot**, which is why an
@@ -273,35 +334,10 @@ export class InstallJob {
     async #beginInstall(req, own, opts) {
         await this.deps.touch();
         await this.deps.ready();
-        /**
-         * One install at a time — the hazard is displacement.
-         *
-         * `repo_clone` calls this and a retried chunk calls it again. Every call
-         * spawns with the same {@link INSTALL_EXEC_ID}, so without this each would
-         * displace the last while the displaced command's drain stayed attached
-         * through `ctx.waitUntil` — then wrote *its* outcome over a record
-         * describing an install still running perfectly well.
-         *
-         * {@link state} rather than the lifecycle's raw `read()`, so a `running`
-         * record left by a dead isolate is resolved here rather than blocking a
-         * legitimate retry forever.
-         *
-         * `takeOverArmedAt` is the one exemption, narrow on purpose: the alarm's
-         * placeholder is a `running` record for an install that has not started, so
-         * the alarm must pass its own guard and only its own. Matching the exact
-         * `startedAt` it wrote is what stops that becoming "take over any running
-         * install", which is displacement again in a new hat.
-         */
+        // {@link state} rather than the lifecycle's raw `read()`: a `running` record
+        // left by a dead isolate is repaired on the way past, and one nobody is
+        // draining is re-attached.
         const current = await this.state();
-        const claim = this.#job.claim(current, this.deps.timeoutMs(), opts?.takeOverArmedAt);
-        if (!claim.ok) {
-            console.info(`[${this.deps.tag()}] an install is already in flight`, {
-                id: this.deps.id(),
-                command: claim.current.command,
-                seconds: Math.round((Date.now() - claim.current.startedAt) / 1000)
-            });
-            return claim.current;
-        }
         /**
          * A full workspace refuses the install and **writes nothing**.
          *
@@ -325,11 +361,126 @@ export class InstallJob {
             });
             return current;
         }
+        /**
+         * One install at a time — the hazard is displacement.
+         *
+         * `repo_clone` calls this and a recovered turn calls it again. Every call
+         * spawns with the same {@link INSTALL_EXEC_ID}, so without this each would
+         * displace the last while the displaced command's drain stayed attached
+         * through `ctx.waitUntil` — then wrote *its* outcome over a record
+         * describing an install still running perfectly well.
+         *
+         * Reserved rather than checked: everything between here and the spawn —
+         * probing the checkout, hashing the lockfile — awaits the container, and a
+         * check that wrote nothing would let a second caller through in that time.
+         *
+         * `takeOverArmedAt` is the one exemption, narrow on purpose: the alarm's
+         * placeholder is a `running` record for an install that has not started, so
+         * the alarm must pass its own guard and only its own. Matching the exact
+         * `startedAt` it wrote is what stops that becoming "take over any running
+         * install", which is displacement again in a new hat.
+         */
+        const reserved = await this.#job.reserve({ command: RESOLVING }, this.deps.timeoutMs(), opts?.takeOverArmedAt);
+        if (!reserved.ok) {
+            console.info(`[${this.deps.tag()}] an install is already in flight`, {
+                id: this.deps.id(),
+                command: reserved.current.command,
+                seconds: Math.round((Date.now() - reserved.current.startedAt) / 1000)
+            });
+            return reserved.current;
+        }
+        const { startedAt } = reserved;
+        const run = this.#job.generation(startedAt);
+        this.#owned.add(startedAt);
+        let handed = false;
+        try {
+            return await this.#spawnReserved(req, run, reserved, own, () => {
+                handed = true;
+            });
+        }
+        finally {
+            if (!handed)
+                this.#owned.delete(startedAt);
+        }
+    }
+    /**
+     * Everything after the reservation: prepare, spawn, and hand the command to
+     * `own`. Split out so {@link #beginInstall} can hold the generation for exactly
+     * this long — `handing` is called as the drain takes it over.
+     */
+    async #spawnReserved(req, run, reserved, own, handing) {
+        const { startedAt } = reserved;
         // Nothing trusts the interception CA here, deliberately: the workspace's own
         // `ready` does it, above every early return in this method. An install
         // already in flight and a full workspace both return before this line, and
         // neither of them means the container cannot speak TLS — the full workspace
         // least of all, since egress is what the agent needs to dig itself out.
+        let prepared;
+        try {
+            prepared = await this.#prepare(req, run, startedAt);
+        }
+        catch (err) {
+            await this.#release(run, reserved.previous, err);
+            throw err;
+        }
+        if (prepared.kind === "settled")
+            return prepared.state;
+        const { resolution, fingerprint, state } = prepared;
+        let handle;
+        try {
+            // The marker lets a new isolate check {@link CONTAINER_TREE_KEY} against
+            // the container; see `reconcile`.
+            const marked = `${resolution.command} && printf %s ` +
+                `${shellQuote(fingerprint ?? "none")} > node_modules/${INSTALLED_MARKER}`;
+            handle = await this.deps.workspace().runtime.exec(marked, {
+                id: INSTALL_EXEC_ID,
+                cwd: req.dir,
+                encoding: "utf8",
+                timeoutMs: this.deps.timeoutMs()
+            });
+        }
+        catch (err) {
+            // The command never started, so nothing will ever drain it and no
+            // re-attach can find it. Close the record here: a `failed` install is
+            // recoverable — the agent is told what happened and can run the command
+            // itself — where a `running` one that nobody owns is not.
+            //
+            // Which sentence, though, depends on why. The default advice — run it
+            // yourself with `bash` — is good for a container that is merely
+            // unreachable, and useless for a deployment fault: the same container is
+            // what `bash` would have to reach. Under one of those, say what an
+            // operator has to do and do not send the agent after a command that
+            // cannot run either.
+            const fault = deploymentFault(err);
+            console.error(`[${this.deps.tag()}] ${fault?.summary ?? "the install could not be started"}`, {
+                id: this.deps.id(),
+                command: resolution.command,
+                ...(fault ? { remedy: fault.remedy } : {}),
+                err: String(err)
+            });
+            const failed = {
+                state: "failed",
+                command: resolution.command,
+                finishedAt: Date.now(),
+                error: fault
+                    ? `the install could not be started: ${fault.remedy}`
+                    : `the install could not be started (${String(err)}). The container ` +
+                        "was most likely unreachable. Run the command yourself with bash, " +
+                        "or clone again to retry it."
+            };
+            return await this.#settle(run, failed);
+        }
+        handing();
+        await own(handle, startedAt);
+        return state;
+    }
+    /**
+     * Everything between a reservation and the spawn: resolve the command, and
+     * settle the run here when there is nothing to install or the tree is already
+     * this lockfile's. Otherwise write this install's own `running` record and
+     * context, and arm the watchdog.
+     */
+    async #prepare(req, run, startedAt) {
         const probe = this.#probe();
         const resolution = await resolveInstallCommand(probe, req.dir, this.deps.plan(), req.repo);
         if (resolution.kind === "skip") {
@@ -337,7 +488,6 @@ export class InstallJob {
                 state: "skipped",
                 reason: resolution.reason
             };
-            await this.#job.write(state);
             /**
              * The common branch, and the one whose absence is indistinguishable from
              * never having been called.
@@ -355,7 +505,7 @@ export class InstallJob {
                 ...(req.repo ? { repo: req.repo } : {}),
                 reason: resolution.reason
             });
-            return state;
+            return { kind: "settled", state: await this.#settle(run, state) };
         }
         const fingerprint = await installFingerprint(probe, req.dir, resolution);
         // Skipped only for this container's own tree, for this lockfile.
@@ -371,18 +521,18 @@ export class InstallJob {
                 ms: 0,
                 tail: "dependencies already installed in this container for this lockfile"
             };
-            await this.#job.write(state);
-            return state;
+            return { kind: "settled", state: await this.#settle(run, state) };
         }
         // The tree is rebuilt from here, so nothing vouches for it until exit 0.
         await this.deps.storage.delete([CONTAINER_TREE_KEY, ...RETIRED_KEYS]);
-        const startedAt = Date.now();
         const state = {
             state: "running",
             command: resolution.command,
             startedAt
         };
-        await this.#job.write(state);
+        if (!(await run.write(state))) {
+            return { kind: "settled", state: await this.#job.read() };
+        }
         await this.#job.putContext({
             dir: req.dir,
             // Kept so a reinstall the alarm drives — which has no caller to ask —
@@ -391,8 +541,7 @@ export class InstallJob {
             // on every later run, installing a different tree than the first time.
             ...(req.repo ? { repo: req.repo } : {}),
             fingerprint,
-            command: resolution.command,
-            startedAt
+            command: resolution.command
         });
         /**
          * Armed **before** the spawn — the hazard is an isolate that dies between
@@ -408,54 +557,33 @@ export class InstallJob {
          * `running`, so finishing first costs one wake-up.
          */
         await this.#job.armWatch();
-        let handle;
-        try {
-            // The marker lets a new isolate check {@link CONTAINER_TREE_KEY} against
-            // the container; see `reconcile`.
-            const marked = `${resolution.command} && printf %s ` +
-                `${shellQuote(fingerprint ?? "none")} > node_modules/${INSTALLED_MARKER}`;
-            handle = await this.deps.workspace().runtime.exec(marked, {
-                id: INSTALL_EXEC_ID,
-                cwd: req.dir,
-                encoding: "utf8",
-                timeoutMs: this.deps.timeoutMs()
-            });
-        }
-        catch (err) {
-            // The command never started, so nothing will ever drain it and no
-            // re-attach can find it. Close the record here: a `failed` install is
-            // recoverable — the subagent is told what happened and can run the command
-            // itself — where a `running` one that nobody owns is not.
-            //
-            // Which sentence, though, depends on why. The default advice — run it
-            // yourself with `sb_exec` — is good for a container that is merely
-            // unreachable, and useless for a deployment fault: the same container is
-            // what `sb_exec` would have to reach. Under one of those, say what an
-            // operator has to do and do not send the subagent after a command that
-            // cannot run either.
-            const fault = deploymentFault(err);
-            console.error(`[${this.deps.tag()}] ${fault?.summary ?? "the install could not be started"}`, {
-                id: this.deps.id(),
-                command: resolution.command,
-                ...(fault ? { remedy: fault.remedy } : {}),
-                err: String(err)
-            });
-            const failed = {
+        return { kind: "spawn", resolution, fingerprint, state };
+    }
+    /** Settle `run` with `state`, or say what stands if it was displaced. */
+    async #settle(run, state) {
+        return (await run.write(state)) ? state : await this.#job.read();
+    }
+    /**
+     * Hand a reservation back when nothing was spawned under it: the record as it
+     * was, unless that was a run of its own — the alarm's placeholder — which
+     * nothing would now finish, so `failed`.
+     */
+    async #release(run, previous, err) {
+        const back = previous.state === "running"
+            ? {
                 state: "failed",
-                command: resolution.command,
+                command: previous.command,
                 finishedAt: Date.now(),
-                error: fault
-                    ? `the install could not be started: ${fault.remedy}`
-                    : `the install could not be started (${String(err)}). The container ` +
-                        "was most likely unreachable. Run the command yourself with sb_exec, " +
-                        "or clone again to retry it."
-            };
-            await this.#job.write(failed);
-            await this.#job.clearWatch();
-            return failed;
+                error: `the install could not be prepared (${String(err)}). Run the ` +
+                    "command yourself with bash, or clone again to retry it."
+            }
+            : previous;
+        try {
+            await run.write(back);
+        }
+        catch (rollback) {
+            throw new AggregateError([err, rollback], "the install could not be prepared, and its reservation could not be released");
         }
-        await own(handle, startedAt);
-        return state;
     }
     /**
      * Whether this container holds a finished tree, asked only when it qualifies a
@@ -504,83 +632,102 @@ export class InstallJob {
                 finishedAt: Date.now(),
                 error: `the install has been running for ${minutes} minutes without ` +
                     "reporting, which is past its timeout — it is not going to finish. " +
-                    "Run the command yourself with sb_exec if you still need it."
+                    "Run the command yourself with bash if you still need it."
             };
-            await this.#job.write(failed);
-            await this.#job.clearWatch();
-            return failed;
+            // Under the record's own generation, so a reservation that took the record
+            // since it was read is not overwritten.
+            const repaired = await this.#job
+                .generation(state.startedAt)
+                .write(failed);
+            return repaired ? failed : await this.#job.read();
         }
-        if (!this.#draining) {
-            await this.#reattachInstall();
-            return await this.#job.read();
-        }
-        return state;
+        if (await this.#drivenHere(state.startedAt))
+            return state;
+        await this.#reattachInstall(state.startedAt);
+        return await this.#job.read();
     }
     /**
      * When a reinstall was queued, if one is waiting to run.
      *
-     * Read by {@link file://../advisory.ts deriveAdvisories}, which is the only
+     * Read by {@link file://./advisory.ts deriveAdvisories}, which is the only
      * caller that needs it: a `failed` record with a repair already queued is a
      * transient condition wearing a permanent record's clothes.
      */
     async reinstallArmedAt() {
         return await this.#job.armedAt();
     }
-    /** True while this isolate holds the drain, so the watchdog leaves it alone. */
-    #draining = false;
-    async #drainInstall(handle, 
     /**
-     * The stamp the install this drain was handed belongs to.
+     * The generations this isolate is driving: reserved, taken over from the
+     * alarm's placeholder, or draining — until the drain ends.
      *
-     * Passed in rather than read below, and that is the whole of the fix: this
-     * method's first `await` is reached long after `#beginInstall` returned, so a
-     * second install that claimed in between has already rewritten the context.
-     * A drain reading it then adopts the *other* install's generation, passes
-     * every `stillMine()` check, and writes its own verdict over a command that
-     * is still running.
+     * The re-attach is for a `running` record nobody here is driving, which is one
+     * a dead isolate left. Asked of a generation that is reserved but not yet
+     * spawned, it finds no exec, or the last container's, and writes `failed` over
+     * an install about to start, whose own verdict is then discarded as
+     * superseded. Owning from the reservation rather than from the drain is what
+     * closes that window: resolving and spawning await the container for seconds.
+     */
+    #owned = new Set();
+    /**
+     * Whether a `running` record's install is this isolate's to finish: one it
+     * owns, or the alarm's placeholder still waiting for the alarm, whose
+     * `startedAt` is the arming stamp.
+     */
+    async #drivenHere(startedAt) {
+        return (this.#owned.has(startedAt) || (await this.#job.armedAt()) === startedAt);
+    }
+    async #drainInstall(handle, 
+    /**
+     * The generation this drain writes under: the `startedAt` its reservation
+     * returned, or — re-attached — the running record's.
      *
-     * Absent only for {@link InstallJob.#reattachInstall}, which by definition
-     * did not start what it is picking up and has nothing but the record to go
-     * on.
+     * Passed in rather than read below: this method's first `await` is reached
+     * long after `#beginInstall` returned, and a record read then may already be
+     * a later install's, whose generation this drain would adopt.
      */
-    ownedAt) {
-        this.#draining = true;
+    startedAt) {
+        this.#owned.add(startedAt);
         const context = await this.#job.context();
         const command = context?.command ?? "(unknown)";
-        const startedAt = ownedAt ?? context?.startedAt ?? Date.now();
         /**
          * Whether this drain still owns the record.
          *
-         * The guard in `startInstall` stops two installs overlapping in the first
-         * place; this makes it harmless if one ever does. A drain can outlive the
-         * command it was watching — `ctx.waitUntil` keeps running after the RPC
+         * The reservation in `#beginInstall` stops two installs overlapping in the
+         * first place; this makes it harmless if one ever does. A drain can outlive
+         * the command it was watching — `ctx.waitUntil` keeps running after the RPC
          * returns — and the damage a late one does is silent: it writes a verdict
          * about a finished command over a record describing a live one, and every
-         * `sb_exec` then reads a result that belongs to nothing.
-         *
-         * `startedAt` is the generation marker. `#beginInstall` rewrites the context
-         * before it spawns, so a drain whose stamp no longer matches has been
-         * superseded and has nothing useful left to say. The marker also **latches**:
-         * ownership is not recoverable, so a stamp that happens to match again does
-         * not hand the record back.
+         * `bash` then reads a result that belongs to nothing.
          *
-         * Wrapped rather than used bare only to log the transition, and only once —
-         * a superseded drain asks this on both the success and the error path.
+         * Every write goes through `run.write`, which checks and writes in one step
+         * and clears the watchdog with a verdict. `stillMine` is asked only before
+         * this drain's own side effects. The superseded line is logged once — a
+         * superseded drain can be refused on both the success and the error path.
          */
-        const generation = this.#job.generation(startedAt);
+        const run = this.#job.generation(startedAt);
         let logged = false;
+        const superseded = async () => {
+            if (logged)
+                return;
+            logged = true;
+            const current = await this.#job.read();
+            console.warn(`[${this.deps.tag()}] discarding a superseded install drain`, {
+                id: this.deps.id(),
+                command,
+                startedAt,
+                current: current.state === "running" ? current.startedAt : current.state
+            });
+        };
         const stillMine = async () => {
-            if (await generation.stillMine())
+            if (await run.stillMine())
                 return true;
-            if (!logged) {
-                logged = true;
-                console.warn(`[${this.deps.tag()}] discarding a superseded install drain`, {
-                    id: this.deps.id(),
-                    command,
-                    startedAt,
-                    current: (await this.#job.context())?.startedAt
-                });
-            }
+            await superseded();
+            return false;
+        };
+        const settle = async (verdict, also) => {
+            if (await run.write(verdict, also))
+                return true;
+            await superseded();
             return false;
         };
         try {
@@ -593,33 +740,48 @@ export class InstallJob {
             if (!(await stillMine()))
                 return;
             if (result.exitCode === 0) {
-                if (context?.dir) {
-                    const tree = {
+                // With the verdict, in the same write: the tree is this install's only if
+                // the verdict is.
+                const tree = context?.dir
+                    ? {
                         dir: context.dir,
                         fingerprint: context.fingerprint,
                         at: Date.now()
-                    };
-                    await this.deps.storage.put(CONTAINER_TREE_KEY, tree);
-                }
+                    }
+                    : undefined;
                 console.info(`[${this.deps.tag()}] install finished`, {
                     id: this.deps.id(),
                     command,
                     seconds: Math.round((Date.now() - startedAt) / 1000)
                 });
-                await this.#job.write({
+                const installed = tree?.fingerprint
+                    ? { dir: tree.dir, fingerprint: tree.fingerprint, at: tree.at }
+                    : undefined;
+                const kept = await settle({
                     state: "done",
                     command,
                     exitCode: 0,
                     finishedAt: Date.now(),
                     ms: Date.now() - startedAt,
                     tail
-                });
+                }, tree
+                    ? {
+                        ...(installed ? this.deps.installedRecords(installed) : {}),
+                        [CONTAINER_TREE_KEY]: tree
+                    }
+                    : undefined);
+                // Outside the verdict: a host that cannot act on the news must not turn
+                // a finished install into a failed one, which the catch below would.
+                if (kept && installed)
+                    await this.deps.installed(installed).catch((err) => {
+                        console.warn(`[${this.deps.tag()}] could not act on a finished install`, { id: this.deps.id(), err: String(err) });
+                    });
             }
             else {
                 // Logged, and this line is not optional. This is the *ordinary* way an
                 // install fails — the other paths are all exceptional — and it used to
                 // write the record and say nothing, so an operator looking at why the
-                // subagent was complaining found the complaint and no cause. The tail
+                // agent was complaining found the complaint and no cause. The tail
                 // is the install's own last words; without it the only copy is inside a
                 // Durable Object nobody can query.
                 console.error(`[${this.deps.tag()}] install failed`, {
@@ -629,7 +791,7 @@ export class InstallJob {
                     seconds: Math.round((Date.now() - startedAt) / 1000),
                     tail: truncateOutput(tail, 1000)
                 });
-                await this.#job.write({
+                await settle({
                     state: "failed",
                     command,
                     finishedAt: Date.now(),
@@ -658,7 +820,7 @@ export class InstallJob {
                 seconds: Math.round((Date.now() - startedAt) / 1000),
                 err: String(err)
             });
-            await this.#job.write({
+            await settle({
                 state: "failed",
                 command,
                 finishedAt: Date.now(),
@@ -666,13 +828,8 @@ export class InstallJob {
             });
         }
         finally {
-            this.#draining = false;
+            this.#owned.delete(startedAt);
             handle[Symbol.dispose]();
-            // Not if this drain was superseded: the watchdog belongs to whichever
-            // install owns the record now, and clearing it here would disarm the one
-            // recovery path the *live* install has.
-            if (!generation.superseded())
-                await this.#job.clearWatch();
         }
     }
     /**
@@ -681,10 +838,12 @@ export class InstallJob {
      * `getExec` with `resume: "tail"` re-opens the stream of a command that is
      * still running in the container — or replays the end of one that finished
      * while nobody was listening, which is the case that would otherwise leave the
-     * record stuck at `running` and every `sb_exec` blocked behind it.
+     * record stuck at `running` and every `bash` blocked behind it.
      */
-    async #reattachInstall() {
-        if (this.#draining)
+    async #reattachInstall(
+    /** The running record's `startedAt`: the generation it picks up. */
+    startedAt) {
+        if (await this.#drivenHere(startedAt))
             return;
         try {
             const handle = await this.deps
@@ -693,12 +852,12 @@ export class InstallJob {
                 encoding: "utf8",
                 resume: "tail"
             });
-            this.deps.waitUntil(this.#drainInstall(handle));
+            this.deps.waitUntil(this.#drainInstall(handle, startedAt));
         }
         catch (err) {
             // The exec is gone entirely — the container was replaced under it. Say so
             // rather than leaving the gate closed forever; the next checkout starts a
-            // new install, and `sb_exec` can run in the meantime.
+            // new install, and `bash` can run in the meantime.
             if (execWasLost(err))
                 await this.deps.containerGone();
             // Same split as the spawn path: a replaced container is worth re-running
@@ -717,16 +876,15 @@ export class InstallJob {
                     err: String(err)
                 });
             const context = await this.#job.context();
-            await this.#job.write({
+            await this.#job.generation(startedAt).write({
                 state: "failed",
                 command: context?.command ?? "(unknown)",
                 finishedAt: Date.now(),
                 error: fault
                     ? `the install stopped without reporting: ${fault.remedy}`
                     : "the install stopped without reporting — its container was most " +
-                        "likely replaced. Re-run it with sb_exec, or clone again to restart it."
+                        "likely replaced. Re-run it with bash, or clone again to restart it."
             });
-            await this.#job.clearWatch();
         }
     }
     /**
@@ -734,24 +892,34 @@ export class InstallJob {
      *
      * `clearArmed` first and unconditionally: this handler runs for minutes, and
      * the stamp left in place is what a second arming would recognise as its own.
-     * `startInstall`'s in-flight guard would catch a double, but the cheaper
+     * The reservation in `#beginInstall` would catch a double, but the cheaper
      * answer is not to schedule one.
      */
     async onRun() {
         const context = await this.#job.context();
         const armedAt = await this.#job.armedAt();
-        await this.#job.clearArmed();
-        if (!context?.dir || armedAt === undefined)
-            return;
-        const state = await this.#awaited({
-            dir: context.dir,
-            ...(context.repo ? { repo: context.repo } : {})
-        }, armedAt);
-        console.info(`[${this.deps.tag()}] armed reinstall finished`, {
-            id: this.deps.id(),
-            dir: context.dir,
-            state: state.state
-        });
+        // Owned before the arm is cleared: from then until the reservation takes it
+        // over, the placeholder is a `running` record that only this call will finish.
+        if (armedAt !== undefined)
+            this.#owned.add(armedAt);
+        try {
+            await this.#job.clearArmed();
+            if (!context?.dir || armedAt === undefined)
+                return;
+            const state = await this.#awaited({
+                dir: context.dir,
+                ...(context.repo ? { repo: context.repo } : {})
+            }, armedAt);
+            console.info(`[${this.deps.tag()}] armed reinstall finished`, {
+                id: this.deps.id(),
+                dir: context.dir,
+                state: state.state
+            });
+        }
+        finally {
+            if (armedAt !== undefined)
+                this.#owned.delete(armedAt);
+        }
     }
     /** The watchdog: an install still running that nobody is draining. */
     async onWatch() {
@@ -760,7 +928,7 @@ export class InstallJob {
             return;
         // Still running and nobody draining it: this isolate is new since the
         // command started. Re-attach, and come back if it is still going.
-        await this.#reattachInstall();
+        await this.#reattachInstall(state.startedAt);
         await this.#job.armWatch();
     }
 }