@cotal-ai/delivery 0.57.0 → 0.59.0

package/dist/delivery.js

@@ -1,11 +1,12 @@
-import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, writeFileSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
-import { CotalEndpoint, DEFAULT_SERVER, LEASE_TTL_MS, accountFromCreds, credsClaims, deliveryBucket, dialerFor, idFromCreds, defaultProbeTimeoutMs, isCasLoss, isPermissionDenied, isReachable, leaseKey, mintCreds, newIdentity, formatSecretStoreIdentity, parseSecretStoreIdentity, sameSecretStoreIdentity, standaloneConnectOpts, startTimerWriter, } from "@cotal-ai/core";
+import { CotalEndpoint, DEFAULT_SERVER, LEASE_TTL_MS, accountFromCreds, credsClaims, deliveryBucket, dialerFor, idFromCreds, isCasLoss, isPermissionDenied, isReachable, leaseKey, mintCreds, newIdentity, formatSecretStoreIdentity, parseSecretStoreIdentity, sameSecretStoreIdentity, standaloneConnectOpts, startTimerWriter, } from "@cotal-ai/core";
 import { PermissionViolationError } from "@nats-io/transport-node";
-import { DELIVERY_CREDS_KIND, DELIVERY_PIDFILE, FsSecretStore, authDir, canonicalLocalProcessPath, canonicalRoot, deliveryCredsKey, findCotalRoot, isWorkspaceTargetError, loadSpaceAuth, reclaimDeadPreUpgradeRecord, removeIdentityPin, resolveMeshTarget, segmentedKey, soleSpaceOf, spaceSegment, workspaceSecretStore, writeIdentityPin } from "@cotal-ai/workspace";
+import { DELIVERY_CREDS_KIND, DELIVERY_PIDFILE, FsSecretStore, authDir, canonicalLocalProcessPath, canonicalRoot, deliveryCredsKey, findCotalRoot, isWorkspaceTargetError, loadSpaceAuth, reclaimDeadPreUpgradeRecord, removePidPair, requireCotalRoot, resolveMeshTarget, segmentedKey, soleSpaceOf, spaceSegment, workspaceSecretStore, writePidPair } from "@cotal-ai/workspace";
 import { startMembership } from "./membership.js";
-import { mayServeOn, brokerGoneVerdict, classifyProbe, DescheduleSampler, leaseAction, LoopLagMeter, PROBE_INTERVAL_MS, PROBE_LATE_FACTOR } from "./watchdog.js";
-import { executeEviction, executePlaneLiveness, executePrincipalLiveness, validateScanTargetAdmission } from "./evict-exec.js";
+import { mayServeOn, leaseAction } from "./watchdog.js";
+import { DeliveryTransportHealth } from "./transport-health.js";
+import { executeEviction, executeEvictions, executePlaneLiveness, executePrincipalLiveness, validateScanTargetAdmission } from "./evict-exec.js";
 /** Re-exported for hosted compositions: the daemon cred's KIND, and the builder that turns it into
  *  the {@link SecretStore} key for a space. Defined once in workspace (the layout is the workspace's)
  *  so the writer, the renewal owner, and this reader can never drift apart. As of P7 the kind is NOT
@@ -106,7 +107,7 @@ export function assertUninjectedCredsSharesCwdRoot(opts) {
     const credsIdentity = { kind: "fs", root: credsWorkspace };
     if (sameSecretStoreIdentity(credsIdentity, cwdIdentity))
         return;
-    throw new Error(`delivery: --creds names workstation ${formatSecretStoreIdentity(credsIdentity)} while membership-rw resolves under ${formatSecretStoreIdentity(cwdIdentity)} (process cwd). Pass both the same workstation root, or inject one SecretStore.`);
+    throw new Error(`delivery: --creds names workstation ${formatSecretStoreIdentity(credsIdentity)} while membership-rw resolves under ${formatSecretStoreIdentity(cwdIdentity)} (${opts.via ?? "process cwd"}). Pass both the same workstation root, or inject one SecretStore.`);
 }
 /**
  * THE ORDERING-CRITICAL HALF of the cred-source decision, split out so it cannot drift below the
@@ -124,6 +125,28 @@ function assertNoLocalCredSourceFlags(v, injected) {
     if (local.length)
         throw new Error(`delivery: ${local.map((f) => `--${f}`).join(" and ")} cannot be combined with an injected secret store — the store is the cred's only source`);
 }
+/** The workspace root a workstation daemon serves, chosen ONCE at start (#723). `--root` names it;
+ *  otherwise it is the nearest `.cotal/` above the working directory. With neither, the daemon refuses
+ *  here and names where it searched: the start directory is not a root anyone set up, and reading the
+ *  cred, the registry and the $SYS pair from it only failed later, after the dial, on a symptom. An
+ *  injected store is the composition's only credential source, so there the root holds no trust
+ *  material and a missing `.cotal/` stays allowed, as {@link recordDeliveryPid} documents. */
+function daemonRoot(v, injected) {
+    if (v.root !== undefined) {
+        const root = resolve(v.root);
+        if (!existsSync(join(root, ".cotal")))
+            throw new Error(`delivery: --root ${v.root} holds no .cotal/ - name the directory that contains the workspace's .cotal/`);
+        return root;
+    }
+    if (injected)
+        return findCotalRoot();
+    try {
+        return requireCotalRoot();
+    }
+    catch (e) {
+        throw new Error(`delivery: ${e.message} - run the daemon from inside its workspace, or name the root with --root <dir>`);
+    }
+}
 /** Where the daemon's pre-minted cred lives — exactly ONE source: an injected {@link SecretStore}
  *  (a hosted composition), an explicit `--creds <file>` as an FS store over that exact file
  *  (uninjected `--creds` that names one real workstation while process cwd
@@ -139,7 +162,7 @@ function assertNoLocalCredSourceFlags(v, injected) {
  *  The injected arm builds the key without touching a filesystem it does not have. `--creds` names
  *  ONE file and is per-space by the operator's own choice of path, so it is neither segmented nor
  *  migrated — the store there is rooted at that file's own directory. */
-function resolveCredsStore(v, space, injected) {
+function resolveCredsStore(v, space, root, injected) {
     if (injected) {
         const key = segmentedKey(DELIVERY_CREDS_KIND, space);
         return {
@@ -162,7 +185,6 @@ function resolveCredsStore(v, space, injected) {
             identity: reloadStoreIdentityFromCredsPath(p, space),
         };
     }
-    const root = findCotalRoot();
     const key = deliveryCredsKey(space, { injected: false, root });
     return {
         store: workspaceSecretStore(root),
@@ -183,7 +205,7 @@ function resolveCredsStore(v, space, injected) {
  *  dev run with no stored cred can opt into `--dev-mint`, which loads the local signer and self-remints
  *  a scoped `delivery` cred (one stable identity) — LOUDLY flagged as dev-only, never the production
  *  contract. */
-async function loadDeliveryCreds(src, v) {
+async function loadDeliveryCreds(src, v, root) {
     const { store, key, where } = src;
     const initial = await store.get(key);
     if (initial !== undefined) {
@@ -213,7 +235,7 @@ async function loadDeliveryCreds(src, v) {
         throw new Error(`delivery: no cred in the injected secret store under key "${key}" — the hosted composition must put it before starting the daemon (local sources are never consulted when a store is injected). The key is PER-SPACE as of P7; a cred put under the bare kind "${DELIVERY_CREDS_KIND}" is not read here.`);
     if (v["dev-mint"] !== undefined) {
         // Space-blind dev path: the root must name exactly one space (soleSpaceOf fails loud on several).
-        const devRoot = authDir(findCotalRoot());
+        const devRoot = authDir(root);
         const devSpace = soleSpaceOf(devRoot);
         const auth = devSpace ? loadSpaceAuth(devRoot, devSpace) : undefined;
         if (!auth)
@@ -258,15 +280,11 @@ export function recordDeliveryPid(root, space) {
     reclaimDeadPreUpgradeRecord(DELIVERY_PIDFILE, ctx);
     const pidPath = canonicalLocalProcessPath(DELIVERY_PIDFILE, ctx);
     const mine = String(process.pid);
-    writeFileSync(pidPath, mine);
-    // #969: pin the pid to its process start so a later teardown can refuse a reused pid.
-    writeIdentityPin(pidPath, process.pid);
+    // #969/#1238: publish the pair by rename so a later teardown never sees a torn pairing.
+    writePidPair(pidPath, process.pid);
     return () => {
         try {
-            if (readFileSync(pidPath, "utf8").trim() !== mine)
-                return; // a successor's record: not ours to remove
-            removeIdentityPin(pidPath);
-            rmSync(pidPath, { force: true });
+            removePidPair(pidPath, mine); // a successor's record is not ours to remove
         }
         catch {
             /* already gone, or unreadable: leaving a record we cannot prove is ours is the safe error */
@@ -321,7 +339,26 @@ export async function runDelivery(args, store) {
         throw e;
     }
 }
-async function runStartedDelivery(args, store, publishReleaser) {
+/** Start one independently owned delivery context. No cwd, process signal, pidfile, or exit
+ * policy is borrowed from the CLI runner. The caller owns its explicit state path and store. */
+export async function startDeliveryService(inputs) {
+    if (!inputs.context.accountPublicKey || !inputs.context.lifecycleUid || !inputs.space || !inputs.servers || !inputs.stateDir)
+        throw new Error("delivery: hosted context needs an account, lifecycle, space, server and stateDir");
+    if (inputs.store.identity === undefined)
+        throw new Error("delivery: hosted SecretStore must declare a stable identity");
+    const actual = parseSecretStoreIdentity(inputs.store.identity);
+    if (!sameSecretStoreIdentity(actual, inputs.storeIdentity) || actual.kind !== "injected")
+        throw new Error("delivery: hosted SecretStore identity does not match the assigned store identity");
+    let releaseOnStartupFailure;
+    try {
+        return await runStartedDelivery({ values: { space: inputs.space, server: inputs.servers }, positionals: [], raw: [] }, inputs.store, (release) => { releaseOnStartupFailure = release; }, inputs);
+    }
+    catch (error) {
+        await releaseOnStartupFailure?.();
+        throw error;
+    }
+}
+async function runStartedDelivery(args, store, publishReleaser, hosted) {
     const v = args.values;
     const shard = v.shard ? Number(v.shard) : 0;
     const shards = v.shards ? Number(v.shards) : 1;
@@ -333,14 +370,19 @@ async function runStartedDelivery(args, store, publishReleaser) {
     // the source now needs the space that this check must precede; the check is therefore split out
     // and stays here, ahead of everything ambient.
     assertNoLocalCredSourceFlags(v, store);
+    // Every workstation read below (cred, registry, $SYS pair, membership feed, pidfile) uses this one
+    // root. A hosted context names its own state directory and borrows no cwd.
+    const root = hosted?.stateDir ?? daemonRoot(v, store);
     // Space comes from --space (the CLI passes it). Only --dev-mint may derive it from the local signer.
-    const space = v.space ?? (v["dev-mint"] !== undefined ? soleSpaceOf(authDir(findCotalRoot())) : undefined);
+    const space = v.space ?? (v["dev-mint"] !== undefined ? soleSpaceOf(authDir(root)) : undefined);
     if (!space)
         throw new Error("delivery: --space is required (the scoped creds file does not encode it)");
-    const credsSrc = resolveCredsStore(v, space, store);
+    const credsSrc = resolveCredsStore(v, space, root, store);
     if (v.creds !== undefined)
-        assertUninjectedCredsSharesCwdRoot({ injected: credsSrc.injected, credsPath: resolve(v.creds) });
-    const creds = await loadDeliveryCreds(credsSrc, v); // pre-minted scoped cred; NO signer/loadSpaceAuth in this path
+        assertUninjectedCredsSharesCwdRoot({ injected: credsSrc.injected, credsPath: resolve(v.creds), cwdRoot: root, via: v.root !== undefined ? "--root" : "process cwd" });
+    const creds = await loadDeliveryCreds(credsSrc, v, root); // pre-minted scoped cred; NO signer/loadSpaceAuth in this path
+    if (hosted !== undefined && accountFromCreds(creds.initial) !== hosted.context.accountPublicKey)
+        throw new Error("delivery: scoped credential account does not match the assigned hosted context");
     let latestCreds = creds.initial; // freshest renewal — the broker-reachability poll below presents it
     // WHICH BROKER, WORKSTATION COMPOSITION (#756, the deliver half). `cotal ps`/`spawn`/`attach`/
     // `status`/`supervise` all resolve the registered mesh for the space/root and dial its broker;
@@ -358,7 +400,6 @@ async function runStartedDelivery(args, store, publishReleaser) {
     let server = v.server ?? DEFAULT_SERVER;
     let registered;
     if (store === undefined) {
-        const root = findCotalRoot();
         try {
             registered = resolveMeshTarget(root, { space });
         }
@@ -407,9 +448,13 @@ async function runStartedDelivery(args, store, publishReleaser) {
             console.error(registered.origin === "manual" || registered.origin === "catalog"
                 ? `✗ delivery: no broker answered at ${server} - "${space}" is registered here but its mesh is not up; start it where it runs, or \`cotal meshes rm ${space}\` to unregister it`
                 : `✗ delivery: no mesh running at ${server} - mesh "${space}" is recorded at ${registered.root} but not running; run \`cotal up\` there to restart`);
+            if (hosted !== undefined)
+                throw new Error(`delivery: no broker answered at ${server}`);
             process.exit(1);
         }
         console.error(`✗ delivery: can't reach NATS at ${server}. Run: cotal up`);
+        if (hosted !== undefined)
+            throw new Error(`delivery: can't reach NATS at ${server}`);
         process.exit(1);
     }
     // PIN THE SCAN TARGET ONCE, HERE. The $SYS sweeps below resolve an ACCOUNT, and a complete sweep
@@ -424,12 +469,12 @@ async function runStartedDelivery(args, store, publishReleaser) {
     // is known for certain, rather than inferred later by probing the store or sniffing the
     // filesystem, both of which report "workstation" for a hosted daemon and would emit a CLI repair
     // the host cannot run. It selects the REPAIR IDIOM only; failure semantics never fork on it.
-    const scanRoot = findCotalRoot();
     const scanTarget = {
-        root: scanRoot,
+        root,
+        named: v.root !== undefined,
         expectedAccount: accountFromCreds(creds.initial),
         source: store === undefined
-            ? { secrets: workspaceSecretStore(scanRoot), space, injected: false, root: scanRoot }
+            ? { secrets: workspaceSecretStore(root), space, injected: false, root }
             : { secrets: store, space, injected: true },
     };
     // The singleton lease is admission to serve this space, so the daemon must prove its scan tenancy
@@ -456,6 +501,8 @@ async function runStartedDelivery(args, store, publishReleaser) {
         watchPresence: true, // read the roster for @mention resolution …
         registerPresence: false, // … but NEVER publish the daemon onto the roster (it's infra, not a peer)
         card: { id: ownId, name: "delivery", role: "delivery", kind: "endpoint" },
+        transportPingIntervalMs: Number(process.env.COTAL_DELIVERY_PING_INTERVAL_MS) || 2500,
+        transportMaxPingOut: Number(process.env.COTAL_DELIVERY_MAX_PING_OUT) || 2,
     });
     // Both channels: raw connection errors ride `error`, while every condition the endpoint is
     // already surviving — a failed 75% renewal, the passive backstop's "still holds the previous
@@ -464,7 +511,49 @@ async function runStartedDelivery(args, store, publishReleaser) {
     const say = (e) => console.error(`! delivery endpoint: ${e.message}`);
     ep.on("error", say);
     ep.on("warning", say);
-    await ep.start();
+    try {
+        await ep.start();
+    }
+    catch (error) {
+        await ep.stop();
+        throw error;
+    }
+    const BROKER_GONE_MS = Number(process.env.COTAL_DELIVERY_BROKER_GONE_MS) || 15_000;
+    const BROKER_GONE_BACKSTOP_MS = Math.max(BROKER_GONE_MS, Number(process.env.COTAL_DELIVERY_BROKER_GONE_BACKSTOP_MS) || BROKER_GONE_MS * 4);
+    let stopHandler;
+    let hostedStartupFailure;
+    let stopHosted;
+    const health = new DeliveryTransportHealth((reason) => {
+        console.error(reason === "credential-expired"
+            ? "✗ delivery: credential expired without renewal"
+            : reason === "backstop"
+                ? "✗ delivery: broker connection unavailable past backstop (broker unreachable), exiting (coupled to the broker)"
+                : "✗ delivery: broker connection unavailable (broker unreachable), exiting (coupled to the broker)");
+        const cause = reason === "credential-expired" ? "credential expired without renewal" : `broker connection unavailable (${reason})`;
+        if (hosted !== undefined) {
+            hostedStartupFailure ??= new Error(`delivery context stopped during startup: ${cause}`);
+            if (stopHosted !== undefined)
+                stopHosted(cause);
+            else
+                void ep.stop(); // before lease acquisition, no context-owned lease can be released
+            return;
+        }
+        if (stopHandler !== undefined)
+            stopHandler(1, cause);
+        else
+            earlyStop(1);
+    }, () => console.error("! delivery: credential expired, awaiting proved renewal"), BROKER_GONE_MS, BROKER_GONE_BACKSTOP_MS);
+    ep.on("transport", ({ connected }) => {
+        health.transport(connected);
+    });
+    ep.on("error", (e) => {
+        if (e.name === "UserAuthenticationExpiredError" || e.cause?.name === "UserAuthenticationExpiredError") {
+            health.credentialExpired();
+        }
+    });
+    ep.on("creds-adopted", () => {
+        health.adopted();
+    });
     // Acquire the single-flight lease BEFORE binding the loops: a loud refusal-to-bind if another daemon
     // already holds this shard (two clients binding the same durable name SPLIT delivery). The bucket TTL
     // frees a crashed holder's lease so a fresh daemon re-acquires.
@@ -498,6 +587,8 @@ async function runStartedDelivery(args, store, publishReleaser) {
             console.error(`✗ delivery: acquiring the lease for shard ${shard} failed: ${e.message}. Not binding.`);
         }
         await ep.stop();
+        if (hosted !== undefined)
+            throw e;
         process.exit(1);
         return;
     }
@@ -521,6 +612,13 @@ async function runStartedDelivery(args, store, publishReleaser) {
     // it is provably ours, then exit. Once the real `shutdown` is installed it REPLACES this, and
     // `stopping` makes the pair idempotent, so a signal arriving mid-swap cannot run both.
     let stopping = false;
+    let unavailable;
+    let closePromise;
+    let membership;
+    let timerWriter;
+    let timerWriterAttempt;
+    let stopLeaseWatch;
+    let renew;
     /** Removes THIS process's liveness record, once it has one. Declared BEFORE the handlers that
      *  call it and assigned below, so a signal landing between the two is a no-op rather than a
      *  temporal-dead-zone throw: at that instant nothing has been written, so there is nothing to
@@ -531,6 +629,7 @@ async function runStartedDelivery(args, store, publishReleaser) {
         if (stopping)
             return;
         stopping = true;
+        health.stop();
         setTimeout(() => process.exit(code), 2000);
         unrecordPid?.();
         void (async () => {
@@ -547,10 +646,20 @@ async function runStartedDelivery(args, store, publishReleaser) {
             process.exit(code);
         })();
     };
-    const earlySigint = () => earlyStop(0);
-    const earlySigterm = () => earlyStop(0);
-    process.on("SIGINT", earlySigint);
-    process.on("SIGTERM", earlySigterm);
+    stopHandler = earlyStop;
+    // A SIGNALLED STOP NAMES ITSELF BEFORE TEARDOWN (#1443). Every other deliberate exit writes its
+    // reason first; without this line a `cotal down`, a systemd stop or Ctrl-C left the log ending on
+    // routine work, which an operator cannot tell from a silent death.
+    const onSignal = (signal, stop) => () => {
+        console.error(`• delivery: received ${signal}, exiting (space ${space}, shard ${shard})`);
+        stop(0);
+    };
+    const earlySigint = onSignal("SIGINT", earlyStop);
+    const earlySigterm = onSignal("SIGTERM", earlyStop);
+    if (hosted === undefined) {
+        process.on("SIGINT", earlySigint);
+        process.on("SIGTERM", earlySigterm);
+    }
     // THE LIVENESS RECORD GOES HERE, AFTER THE ACQUIRE AND NOT BEFORE IT (#1528).
     //
     // The record answers "which process is this space's delivery daemon", and until this line this
@@ -566,7 +675,8 @@ async function runStartedDelivery(args, store, publishReleaser) {
     // own `ready` flag already carries it; the pidfile only ever claimed a process.
     //
     // Placed one statement after the signal handlers, so every exit path from here on can remove it.
-    unrecordPid = recordDeliveryPid(findCotalRoot(), space);
+    if (hosted === undefined)
+        unrecordPid = recordDeliveryPid(root, space);
     // AND THE SAME RELEASE, REACHABLE BY AN ORDINARY `catch`. The two process-level guards below
     // only see a fault that reaches the RUNTIME. A start-up rejection on the public CLI path does
     // not: `runCli` awaits this function inside its own try/catch (cli/src/command.ts), so the
@@ -577,22 +687,58 @@ async function runStartedDelivery(args, store, publishReleaser) {
     // exists". So the release is published HERE, to a scope that a plain `catch` around the rest of
     // start-up can reach, and the process guards stay as the backstop for faults that never become
     // a rejection this function can see (a synchronous throw in a timer, say).
-    publishReleaser(async () => {
-        if (stopping)
-            return;
+    const close = () => {
+        if (closePromise !== undefined)
+            return closePromise;
         stopping = true;
-        unrecordPid?.(); // the record dies with the daemon it describes, including on a start-up failure
-        try {
-            const own = await ep.readDeliveryLeaseEntry(shard);
-            if (own !== undefined && ep.ownsDeliveryLease(own.info))
-                await ep.releaseDeliveryLease(shard, own.revision);
-        }
-        catch { /* broker may be gone - the bucket TTL is the crash-safe release authority */ }
-        try {
-            await ep.stop();
-        }
-        catch { /* broker may be gone */ }
-    });
+        if (renew !== undefined)
+            clearInterval(renew);
+        health.stop();
+        stopLeaseWatch?.();
+        unrecordPid?.();
+        closePromise = (async () => {
+            try {
+                await ep.quiescePlane3();
+            }
+            catch { /* stop still releases the connection */ }
+            try {
+                const own = await ep.readDeliveryLeaseEntry(shard);
+                if (own !== undefined && ep.ownsDeliveryLease(own.info))
+                    await ep.releaseDeliveryLease(shard, own.revision);
+            }
+            catch { /* broker may be gone - the bucket TTL is the crash-safe release authority */ }
+            try {
+                await membership?.stop();
+            }
+            catch { /* broker may be gone */ }
+            try {
+                await timerWriter?.handle.stop();
+            }
+            catch { /* broker may be gone */ }
+            try {
+                await Promise.race([timerWriter?.nc.drain(), new Promise((r) => setTimeout(r, 1000))]);
+            }
+            catch { /* broker may be gone */ }
+            try {
+                await timerWriterAttempt;
+            }
+            catch { /* startup or broker fault */ }
+            try {
+                await ep.stop();
+            }
+            catch { /* broker may be gone */ }
+        })();
+        return closePromise;
+    };
+    publishReleaser(close);
+    stopHosted = (cause) => {
+        unavailable = `delivery context stopped (code 1): ${cause}`;
+        void close();
+    };
+    if (hostedStartupFailure !== undefined) {
+        await close();
+        throw hostedStartupFailure;
+    }
     // A START-UP FAILURE AFTER THE ACQUIRE MUST ALSO GIVE THE SHARD BACK. Between this point and the
     // handler swap far below, a throw would otherwise propagate out of `runDelivery` with the row
     // still claiming the shard, stranding it for the bucket TTL exactly as an unhandled signal did -
@@ -618,15 +764,21 @@ async function runStartedDelivery(args, store, publishReleaser) {
     };
     const earlyUncaught = (e) => earlyFault("faulted", e);
     const earlyRejection = (e) => earlyFault("rejected a promise", e);
-    process.on("uncaughtException", earlyUncaught);
-    process.on("unhandledRejection", earlyRejection);
+    if (hosted === undefined) {
+        process.on("uncaughtException", earlyUncaught);
+        process.on("unhandledRejection", earlyRejection);
+    }
     // Broker-sourced graph membership handle — declared BEFORE Plane-3 so the delivery-admin reload
     // hook below can close over it (it starts further down; the closure reads it live).
-    let membership;
     // WHY the feed is down, carried from `startMembership` so the adoption refusal below names the real
     // fault instead of its symptom. Without it, an expired $SYS observer cred surfaces to the operator
     // only as "membership feed is not running" (#338).
     let membershipDown;
+    // START-UP IS NOT DONE UNTIL THE LEASE WATCH IS BOUND (#2304). The ready flip below releases
+    // `ensureDelivery`, and so the manager, whose boot renewal pass sends `reloadCreds` at once. The
+    // endpoint refuses that adoption while this is false, rather than reconnecting this connection
+    // underneath the membership start and the lease watch that are still pending on it.
+    let startedUp = false;
     // Host Plane-3 (fan-out writer + trusted reader) AND serve the ctl.delivery runtime durable ops. The
     // reader re-authorizes each entry against the durable ACL registry, read FRESH per entry. The
     // delivery-admin rail's `reloadCreds` (explicit class-2 adoption) also reloads the membership feed's
@@ -647,6 +799,8 @@ async function runStartedDelivery(args, store, publishReleaser) {
         // Live-eviction executor (D5 slice 6): per-call $SYS observer/evictor connections; refuses
         // loudly on a pre-evictor space. Rare repair/flip step — never a standing $SYS conn here.
         evictPrincipal: (principal) => executeEviction(server, scanTarget, principal),
+        // The same executor over a set: one shared sweep for a large credential family's holders.
+        evictPrincipals: (principals) => executeEvictions(server, scanTarget, principals),
         // Plane-claim liveness oracle (#29 HIGH 3): read-only $SYS CONNZ per call; the auth plane's
         // stale-claim reclaim gates on this verdict (any refusal/unknown blocks takeover, fail-closed).
         planeConnLiveness: (query) => executePlaneLiveness(server, scanTarget, query),
@@ -655,14 +809,30 @@ async function runStartedDelivery(args, store, publishReleaser) {
         // killing it to discover it was alive (any refusal/unknown blocks the repair, fail-closed).
         principalLiveness: (principal) => executePrincipalLiveness(server, scanTarget, principal),
         reloadStoreIdentity: () => reloadStoreIdentity,
+        onDeliveryCredsAdopted: () => health.adopted(),
+        startupComplete: () => startedUp,
     });
+    // The peer-readable liveness responder (#1577), bound BEFORE the ready flip below and
+    // deliberately so. It answers from the lease at PROBE time, so during the window between binding
+    // the loops and flipping ready it answers `unbound` — which is the truth, and is the whole point:
+    // a surface that only came up once everything was healthy could never report the unhealthy state
+    // it exists to report. Presence only; the lease row never crosses the wire.
+    ep.serveDeliveryLiveness(shard);
     // Flip the lease to READY only now — after the loops + ctl.delivery responder are bound — so readiness
     // waiters (ensureDelivery) and the cotal_channels health surface see "ready" iff the responder is up,
     // not merely that the single-flight slot was claimed.
     try {
         revision = await ep.markDeliveryLeaseReady(shard, revision);
     }
-    catch { /* lost the lease between acquire and ready — the renew loop's CAS failure will exit us */ }
+    catch (error) {
+        if (hosted !== undefined)
+            throw error;
+        /* CLI: the renew loop's CAS failure will exit us if the lease was lost. */
+    }
+    if (hostedStartupFailure !== undefined) {
+        await close();
+        throw hostedStartupFailure;
+    }
     console.log(`✓ delivery daemon up (space ${space}${shards > 1 ? `, shard ${shard}/${shards}` : ""}) — stop with: cotal down`);
     // Broker-sourced graph membership: a SEPARATE module on its OWN connections (system-account CONNZ
     // reader + data-account feed writer), isolated from Plane-3. Fail-soft — a missing cred / start error
@@ -675,12 +845,23 @@ async function runStartedDelivery(args, store, publishReleaser) {
         // `accountId` is the account THIS daemon's own cred authenticates as (pinned above), not a
         // `.cotal/membership.json` read: the file was never an independent source (same directory as the
         // creds, so a wrong root was wrong for both) and a hosted composition has none.
-        ({ handle: membership, down: membershipDown } = await startMembership({ space, server, accountId: scanTarget.expectedAccount }, store));
+        ({ handle: membership, down: membershipDown } = await startMembership({ space, server, accountId: scanTarget.expectedAccount, root }, store));
     }
     catch (e) {
         membershipDown = e.message;
         console.error(`! membership: failed to start (${membershipDown}); graph membership degraded, delivery unaffected`);
     }
+    // A store read can finish after a startup health fault closed the endpoint. Dispose the feed
+    // it just returned before rejecting; it did not exist when the original close ran.
+    if (hostedStartupFailure !== undefined) {
+        try {
+            await membership?.stop();
+        }
+        finally {
+            await close();
+        }
+        throw hostedStartupFailure;
+    }
     // The TIMER WRITER (SPEC 13.2): the pump that turns workflow `.schedule` requests into armed
     // broker schedules. Hosted here because this daemon is the space's standing server-side process;
     // without a running writer no pause on the space ever expires. Its OWN connection under the same
@@ -688,8 +869,7 @@ async function runStartedDelivery(args, store, publishReleaser) {
     // module-per-connection isolation), re-dialed with the FRESHEST cred by a supervised restart
     // loop: a fault never kills delivery, is never silent (each attempt logs why the space cannot
     // expire pauses right now), and a cred that gains rows at renewal is picked up on the next dial.
-    let timerWriter;
-    void (async () => {
+    timerWriterAttempt = (async () => {
         let backoffMs = 5_000;
         for (;;) {
             if (stopping)
@@ -702,7 +882,16 @@ async function runStartedDelivery(args, store, publishReleaser) {
                     name: "cotal-delivery-timer-writer",
                     maxReconnectAttempts: -1,
                 });
+                if (stopping) {
+                    await nc.close();
+                    return;
+                }
                 const handle = await startTimerWriter(nc, space);
+                if (stopping) {
+                    await handle.stop();
+                    await nc.close();
+                    return;
+                }
                 timerWriter = { handle, nc };
                 backoffMs = 5_000;
                 console.log(`✓ timer writer up (space ${space}) — workflow pauses on this space expire`);
@@ -710,8 +899,13 @@ async function runStartedDelivery(args, store, publishReleaser) {
                 return;
             }
             catch (e) {
-                if (stopping)
+                if (stopping) {
+                    try {
+                        await nc?.close();
+                    }
+                    catch { /* already closed */ }
                     return;
+                }
                 console.error(`! timer writer: down (${e.message}) — retrying in ${Math.round(backoffMs / 1000)}s; pauses on this space do not expire until it is back`);
                 try {
                     await nc?.drain();
@@ -723,17 +917,24 @@ async function runStartedDelivery(args, store, publishReleaser) {
             }
         }
     })();
-    const shutdown = (code) => {
+    const shutdown = (code, cause) => {
+        if (hosted !== undefined) {
+            unavailable = cause ? `delivery context stopped (code ${code}): ${cause}` : `delivery context stopped (code ${code})`;
+            void close();
+            return;
+        }
         if (stopping)
             return;
         stopping = true;
-        clearInterval(renew);
-        clearInterval(brokerWatch);
-        stopLeaseWatch();
+        if (renew !== undefined)
+            clearInterval(renew);
+        health.stop();
+        stopLeaseWatch?.();
         // THE RECORD GOES FIRST, AND SYNCHRONOUSLY. Everything below this line talks to a broker that
         // may be dead and is bounded only by the 2s hard exit; a record left behind because a drain
         // hung is a record that outlives its process, which is this issue's defect re-entering through
-        // the exit path. `rmSync` needs no broker and cannot hang, so it runs before any of it.
+        // the exit path. Removing it needs no broker, and waits only on a publish of the same record
+        // that is in flight (#1238), so it runs before any of it.
         unrecordPid?.();
         // Hard-exit fallback: a graceful release/stop talks to the broker, which may be DEAD (the broker-gone
         // exit path) — don't let that hang the process. Force exit if the graceful path doesn't finish quickly.
@@ -797,6 +998,7 @@ async function runStartedDelivery(args, store, publishReleaser) {
             process.exit(code);
         })();
     };
+    stopHandler = shutdown;
     // SIGNAL HANDLERS GO UP THE MOMENT `shutdown` EXISTS, NOT AFTER STARTUP FINISHES.
     //
     // The lease is acquired long before this point, and registration used to sit at the very end of
@@ -818,16 +1020,18 @@ async function runStartedDelivery(args, store, publishReleaser) {
     // knows nothing of the renew timer, the membership feed or the timer writer - would run first and
     // set `stopping`, making the full teardown a no-op. `stopping` still guards the swap itself, so a
     // signal delivered between these two statements is handled exactly once.
-    process.off("SIGINT", earlySigint);
-    process.off("SIGTERM", earlySigterm);
-    // The start-up fault guards go too, and BY REFERENCE: `removeAllListeners` here would strip
-    // listeners this daemon does not own. They exist to cover the window where no `shutdown` exists;
-    // past this line a fault should surface normally rather than become a quiet exit that skips the
-    // full teardown.
-    process.off("uncaughtException", earlyUncaught);
-    process.off("unhandledRejection", earlyRejection);
-    process.on("SIGINT", () => shutdown(0));
-    process.on("SIGTERM", () => shutdown(0));
+    if (hosted === undefined) {
+        process.off("SIGINT", earlySigint);
+        process.off("SIGTERM", earlySigterm);
+        // The start-up fault guards go too, and BY REFERENCE: `removeAllListeners` here would strip
+        // listeners this daemon does not own. They exist to cover the window where no `shutdown` exists;
+        // past this line a fault should surface normally rather than become a quiet exit that skips the
+        // full teardown.
+        process.off("uncaughtException", earlyUncaught);
+        process.off("unhandledRejection", earlyRejection);
+        process.on("SIGINT", onSignal("SIGINT", shutdown));
+        process.on("SIGTERM", onSignal("SIGTERM", shutdown));
+    }
     /** What the broker says about THIS shard's lease key right now, the verdict a failed renew does
      *  NOT have. `unknown` never collapses into `gone`: not being able to look is not the same fact as
      *  looking and finding nothing (#1318).
@@ -896,6 +1100,8 @@ async function runStartedDelivery(args, store, publishReleaser) {
      *  caller has just proven ownership; `why` is what proved it, since "it started serving again" is
      *  only auditable next to the evidence that permitted it. */
     const resumeServing = async (why) => {
+        if (stopping)
+            return;
         try {
             await ep.rearmPlane3();
         }
@@ -950,7 +1156,7 @@ async function runStartedDelivery(args, store, publishReleaser) {
     // establish it refuses to start, loudly (the await below throws out of start-up and the
     // releaser gives the shard back), rather than degrade into the polled window this issue is
     // about. The stop handle is cleared on every exit path the renew interval is: `shutdown`.
-    const stopLeaseWatch = await ep.watchDeliveryLease(shard, (info) => {
+    stopLeaseWatch = await ep.watchDeliveryLease(shard, (info) => {
         if (stopping)
             return;
         // The row as this event states it: an event for a row we own answers ITSELF (no state
@@ -1009,6 +1215,12 @@ async function runStartedDelivery(args, store, publishReleaser) {
             }
         })().catch((e) => console.error(`! delivery: the lease watch trigger faulted (${e.message}); the renew interval remains the arbiter`));
     });
+    if (hosted !== undefined && stopping) {
+        stopLeaseWatch?.();
+        await close();
+        throw hostedStartupFailure ?? new Error(unavailable ?? "delivery context stopped during startup");
+    }
+    startedUp = true;
     // Renew the lease at ~half the TTL so a healthy holder never self-evicts.
     //
     // A FAILED RENEW IS A QUESTION, NOT A VERDICT (#1318). The shipped code exited on ANY renew
@@ -1021,7 +1233,7 @@ async function runStartedDelivery(args, store, publishReleaser) {
     // second refused over a sequence the first legitimately moved, a conflict this daemon manufactures
     // itself and then reads as someone else's takeover.
     let renewInFlight = false;
-    const renew = setInterval(() => {
+    renew = setInterval(() => {
         if (stopping || renewInFlight)
             return;
         renewInFlight = true;
@@ -1126,187 +1338,19 @@ async function runStartedDelivery(args, store, publishReleaser) {
             }
         })();
     }, Math.max(1000, Math.floor(LEASE_TTL_MS / 2)));
-    // Coupled to the broker: POLL its reachability. Survive brief blips (the endpoint reconnects on its
-    // own), but EXIT if the broker is GONE, the endpoint would otherwise retry reconnect forever (its
-    // terminal-close never fires), so this is what stops the daemon outliving the server it serves.
-    // (`cotal up`/`down` teardown stops it too.) The window is env-overridable for tests.
-    //
-    // WHAT "GONE" MEANS IS NOW DECIDED FROM EVIDENCE, NOT FROM A CLOCK (#1318). Three conditions used
-    // to produce one signal and only one of them was a dead server: the broker being down, this
-    // process being descheduled so the interval never fired, and a probe that could not complete a
-    // handshake because the local process could not get scheduled to finish it. A wall-clock
-    // `Date.now() - lastReachable` cannot tell them apart, and widening it only moves the threshold.
-    // Three signals separate them, and all three are things this process can actually observe:
-    //
-    //   • MEASURED LOOP LAG. The gap between consecutive firings of a timer we own, minus its nominal
-    //     period, is local starvation by direct measurement. It is credited back, so time this process
-    //     spent off the runqueue is never counted against the broker.
-    //   • COMPLETED NEGATIVE PROBES. Only a probe that RAN TO COMPLETION and returned false is
-    //     evidence about the server. A probe that never ran contributes nothing (that was the whole
-    //     defect: the window aged with no probe having failed), and one that REJECTED is an
-    //     unanswered question, it now resets the counter and logs, where it used to be swallowed by
-    //     `.catch(() => {})` and silently age the window.
-    //   • TRANSPORT-LEVEL LIVENESS. A `transport: connected` edge from the endpoint's own connection
-    //     cannot happen without a server on the other end, so it is positive evidence obtained for
-    //     free, on a path that does not need this process to schedule a probe at all.
-    //
-    // A starvation diagnosis therefore reports DEGRADED and keeps serving; a genuinely dead broker
-    // still exits, on the same window, as fast as completed probes can say so.
-    const BROKER_GONE_MS = Number(process.env.COTAL_DELIVERY_BROKER_GONE_MS) || 15_000;
-    // How many completed negatives make elapsed time believable. Two is the floor: one completed
-    // negative is a single refused connect, which a loopback under momentary pressure can produce.
-    // The default scales with the window so a test that shortens the window does not thereby demand
-    // more evidence than the window has room for.
-    const BROKER_GONE_PROBES = Math.max(2, Number(process.env.COTAL_DELIVERY_BROKER_GONE_PROBES) || Math.ceil(BROKER_GONE_MS / PROBE_INTERVAL_MS / 2));
-    // The hard backstop: past this, no amount of measured lag or transport optimism keeps the daemon
-    // alive. A daemon that OUTLIVES a dead broker is worse than one that restarts unnecessarily, so
-    // the repair is bounded and fails toward exiting.
-    const BROKER_GONE_BACKSTOP_MS = Math.max(BROKER_GONE_MS, Number(process.env.COTAL_DELIVERY_BROKER_GONE_BACKSTOP_MS) || BROKER_GONE_MS * 4);
-    let lastReachable = Date.now();
-    let completedNegatives = 0;
-    let degraded = false;
-    // What the endpoint's OWN connection reports about its socket to this same broker. Seeded true
-    // because `ep.start()` above completed, which it cannot do without a server having answered.
-    let transportConnected = true;
-    const lag = new LoopLagMeter(PROBE_INTERVAL_MS);
-    /** Positive evidence, from wherever it came: restart the window and its lag budget together. */
-    const sawBroker = () => {
-        lastReachable = Date.now();
-        completedNegatives = 0;
-        lag.reset();
-        if (degraded) {
-            degraded = false;
-            console.error(`✓ delivery: the broker is answering again, Plane-3 is serving normally (space ${space})`);
-        }
-    };
-    // The endpoint's OWN transport edge. This is evidence the daemon gets without being scheduled to
-    // probe for it, and it is the signal that distinguishes "the connection object reports a
-    // transport-level close" from "silence": a live connection to that address proves a server, while
-    // a disconnect is the endpoint's own business (nats.js reconnects through blips of its own
-    // accord) and merely stops excusing a probe that will not complete.
-    ep.on("transport", (t) => {
-        transportConnected = t.connected;
-        if (t.connected)
-            sawBroker();
-    });
-    // THE BUDGET THE PROBE IS ACTUALLY GIVEN, asked of the one function that decides it. A ws(s)
-    // broker rides an HTTPS edge and gets 5s, not the 1s a loopback TCP broker gets; judging a ws
-    // probe against 1000 would read every honest refusal on such a broker as this process's own
-    // starvation, so the completed-negative count could never rise and a genuinely dead ws broker
-    // would be ended only by the backstop, with the wrong reason in its log. The budget and the
-    // judgment have to come from the same place.
-    const probeBudgetMs = defaultProbeTimeoutMs(server);
-    const brokerWatch = setInterval(() => {
-        if (stopping)
-            return;
-        // Measure FIRST, before any await: this is the gap since the previous firing, and it is the
-        // only place the daemon can learn that it was not scheduled.
-        lag.tick(Date.now());
-        // THE SAME TRANSPORT AS EVERY OTHER DIAL IN THIS PROCESS. This poll carries `latestCreds` — a
-        // standing credential — and `isReachable` performs a real authenticated connect whenever creds
-        // are supplied, every 2 seconds, for the life of the daemon. It is the most repeated credential
-        // presentation in the system, and it was the one dial here that did not name its transport.
-        //
-        // The poll predates TLS and is identical on `main`, where nothing is encrypted and it is at
-        // least consistent. What is new is the ASYMMETRY: with the two dials above upgraded, an operator
-        // who enables TLS would get a protected main path and an unprotected watchdog. An inconsistent
-        // guarantee is worse than a uniformly absent one, because the operator now believes something.
-        const probeStarted = Date.now();
-        // Watch this process's own scheduling FOR THE DURATION OF THE PROBE. A refusal is only evidence
-        // about the server if the server was actually given the time the deadline promised it, and
-        // under short CPU slices most of a probe's wall-clock can be time this process was not running.
-        const sampler = new DescheduleSampler();
-        sampler.start(probeStarted);
-        void isReachable(server, { creds: latestCreds, ...(tls ? { tls: true } : {}) })
-            .then((ok) => classifyProbe(ok, Date.now() - probeStarted, probeBudgetMs, PROBE_LATE_FACTOR, sampler.stop()), 
-        // A REJECTED probe is an unanswered question, not a negative answer. It used to be swallowed
-        // whole by `.catch(() => {})`, so it neither refreshed the window nor evaluated anything and
-        // silently aged the daemon toward an exit it had gathered no evidence for.
-        (e) => {
-            sampler.stop();
-            console.error(`! delivery: the broker probe did not complete (${e.message}), no verdict from it; serving, retrying`);
-            return classifyProbe(undefined, Date.now() - probeStarted);
-        })
-            .then((probe) => {
-            if (stopping)
-                return;
-            if (probe.counts === "positive") {
-                sawBroker();
-                return;
-            }
-            if (probe.counts === "incomplete") {
-                // The run of credible refusals is broken by a probe that did not complete.
-                completedNegatives = 0;
-                return;
-            }
-            if (probe.counts === "starved") {
-                // The probe RAN and said no, but its answer arrived so far past its own deadline that the
-                // deadline was enforced against this process rather than against the server. That is the
-                // starved-client case, and it is the one an elapsed-time predicate cannot see at all: the
-                // timer is firing, the probes are completing, and every one of them is `false`.
-                // Dated with the instant the answer ARRIVED, so the meter can tell whether this stall is
-                // the same wall-clock interval the tick above already charged (union, counted once) or an
-                // adjacent one (disjoint, both counted). The overlap question is settled by timestamps
-                // rather than by comparing magnitudes, which cannot distinguish the two.
-                lag.credit(probe.lateBy, Date.now());
-                completedNegatives = 0;
-            }
-            else {
-                // A refusal that arrived on time. This is the only thing that may accrue against the broker.
-                completedNegatives += 1;
-            }
-            // ONE SPAN, MEASURED ONCE, USED FOR BOTH. Reading the clock twice would let the elapsed span
-            // and the lag clamp disagree by the cost of the call itself.
-            const sinceReachable = Date.now() - lastReachable;
-            const verdict = brokerGoneVerdict({
-                msSinceLastReachable: sinceReachable,
-                // CLAMPED TO THE SPAN IT IS SUBTRACTED FROM. The interval gap and a late probe answer can
-                // testify to the same stall, and the meter cannot tell that from two adjacent stalls, so it
-                // keeps both charges and the bound is applied here, where the span is known. Without it a
-                // 10s stall read 17s and drove unstarved time negative, which cannot be cleared by any
-                // amount of real outage: a dead broker then survives the evidence clause and exits only on
-                // the backstop, 44s -> 62s measured. Time not had cannot exceed time passed.
-                starvedMs: lag.starvedMsWithin(sinceReachable),
-                completedNegatives,
-                transportConnected,
-                windowMs: BROKER_GONE_MS,
-                requiredNegatives: BROKER_GONE_PROBES,
-                backstopMs: BROKER_GONE_BACKSTOP_MS,
-            });
-            if (verdict.exit) {
-                // SAY WHICH EXIT THIS IS. The single message here previously claimed completed probes over
-                // ">15s of unstarved time" on BOTH paths, and on the backstop path that sentence is simply
-                // false: the backstop fires precisely when the evidence clauses did NOT conclude, often
-                // with zero completed refusals. An operator debugging a starved host was handed a
-                // confident evidentiary claim the daemon had not established.
-                console.error(verdict.reason === "backstop"
-                    ? `✗ delivery: giving up on elapsed time alone, ${Math.round(BROKER_GONE_BACKSTOP_MS / 1000)}s since the last ` +
-                        `confirmed reachability with no sufficient evidence either way (${completedNegatives} completed probes refused, ` +
-                        `${Math.round(lag.starvedMsWithin(sinceReachable) / 1000)}s of local scheduler lag credited). This is a BOUND, not a diagnosis: the ` +
-                        `broker may be gone or this process may have been starved past the bound, exiting (coupled to the broker)`
-                    : `✗ delivery: broker unreachable, ${completedNegatives} completed probes refused within their deadline over ` +
-                        `>${BROKER_GONE_MS / 1000}s of unstarved time (${Math.round(lag.starvedMsWithin(sinceReachable) / 1000)}s of local scheduler lag credited), exiting (coupled to the broker)`);
-                shutdown(1);
-                return;
-            }
-            // NOT an exit. Say so once per episode, naming WHICH condition it is, so an operator reading
-            // the log during a load incident sees "this host starved me" rather than a daemon that
-            // silently vanished.
-            if (!degraded && verdict.reason !== "reachable") {
-                degraded = true;
-                console.error(verdict.reason === "starved"
-                    ? `! delivery: DEGRADED, cannot reach the broker, but ${Math.round(lag.starvedMsWithin(sinceReachable) / 1000)}s of that window was local scheduler lag (this host is starving this process, not the broker); serving, retrying`
-                    : verdict.reason === "transport-live"
-                        ? `! delivery: DEGRADED, a fresh probe cannot complete, but this daemon's own connection to ${server} is still open, so the broker is there and this process cannot ask; serving, retrying`
-                        : `! delivery: DEGRADED, the broker has not answered for >${BROKER_GONE_MS / 1000}s but only ${completedNegatives} of ${BROKER_GONE_PROBES} probes have refused within their deadline; serving, retrying`);
-            }
-        })
-            .catch((e) => {
-            // The verdict path itself faulted. Never silent, and never an exit: a bug in the detector
-            // must not end the daemon it is meant to keep alive.
-            console.error(`! delivery: the broker watch tick faulted (${e.message}); serving, retrying`);
-        });
-    }, PROBE_INTERVAL_MS);
-    await new Promise(() => { }); // run until signalled
+    // Native transport health is driven by DeliveryTransportHealth above (resident transport events).
+    if (hosted !== undefined) {
+        const context = hosted.context;
+        return {
+            readiness() {
+                return unavailable !== undefined
+                    ? { state: "unavailable", context, cause: unavailable }
+                    : { state: stopping ? "draining" : "ready", context };
+            },
+            async drain() { await close(); },
+            close,
+        };
+    }
+    await new Promise(() => { }); // CLI runs until signalled
 }
 //# sourceMappingURL=delivery.js.map