@cotal-ai/delivery 0.49.0 → 0.50.0

package/dist/delivery.js

@@ -1,8 +1,9 @@
-import { existsSync } from "node:fs";
+import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
-import { CotalEndpoint, DEFAULT_SERVER, LEASE_TTL_MS, accountFromCreds, credsClaims, dialerFor, idFromCreds, isReachable, mintCreds, newIdentity, formatSecretStoreIdentity, parseSecretStoreIdentity, sameSecretStoreIdentity, standaloneConnectOpts, startTimerWriter, } from "@cotal-ai/core";
-import { DELIVERY_CREDS_KIND, FsSecretStore, authDir, deliveryCredsKey, findCotalRoot, loadSpaceAuth, segmentedKey, soleSpaceOf, spaceSegment, workspaceSecretStore } from "@cotal-ai/workspace";
+import { CotalEndpoint, DEFAULT_SERVER, LEASE_TTL_MS, accountFromCreds, credsClaims, dialerFor, idFromCreds, defaultProbeTimeoutMs, isReachable, mintCreds, newIdentity, formatSecretStoreIdentity, parseSecretStoreIdentity, sameSecretStoreIdentity, standaloneConnectOpts, startTimerWriter, } from "@cotal-ai/core";
+import { DELIVERY_CREDS_KIND, DELIVERY_PIDFILE, FsSecretStore, authDir, canonicalLocalProcessPath, deliveryCredsKey, findCotalRoot, loadSpaceAuth, reclaimDeadPreUpgradeRecord, removeIdentityPin, segmentedKey, soleSpaceOf, spaceSegment, workspaceSecretStore, writeIdentityPin } from "@cotal-ai/workspace";
 import { startMembership } from "./membership.js";
+import { mayServeOn, brokerGoneVerdict, classifyProbe, DescheduleSampler, leaseAction, LoopLagMeter, PROBE_INTERVAL_MS, PROBE_LATE_FACTOR } from "./watchdog.js";
 import { executeEviction, executePlaneLiveness, executePrincipalLiveness, validateScanTargetAdmission } from "./evict-exec.js";
 /** Re-exported for hosted compositions: the daemon cred's KIND, and the builder that turns it into
  *  the {@link SecretStore} key for a space. Defined once in workspace (the layout is the workspace's)
@@ -224,6 +225,53 @@ async function loadDeliveryCreds(src, v) {
     throw new Error(`delivery: no scoped creds at ${where}. Launch via \`cotal setup\`/\`cotal go\` (the setup helper mints + writes it), or pass --creds <file>; for a standalone dev run use --dev-mint.`);
 }
 // Parsing lives in the dispatcher now, driven by the `deliver` command's declared flags.
+/**
+ * RECORD THIS PROCESS as the space's delivery daemon, and un-record it on a clean exit (#1528).
+ *
+ * The daemon writes its OWN record because it is the only participant that always knows it is
+ * running. Until now `delivery.<space>.pid` was written ONLY by the CLI launcher
+ * (`startDeliveryDetached`), so every other route to a live daemon — a container entrypoint,
+ * systemd, an operator typing `cotal deliver --space …`, a hosted composition calling
+ * {@link runDelivery} — left whatever was on disk untouched and readers believed it. A record naming
+ * a pid that died days ago does not merely under-report: `down`'s `mayBeRunning` guard exists to
+ * fail CLOSED so `cotal down nats` cannot pull the broker out from under a live dependant, and a
+ * stale record supplies exactly the proof-of-death that guard requires.
+ *
+ * The launcher's shape is followed rather than re-invented: the CANONICAL path (never a pre-upgrade
+ * name), a provably dead pre-upgrade record reclaimed first so an upgraded root never holds both
+ * spellings, then the #969 identity pin beside it. The pin goes with the record on the way out.
+ *
+ * The removal is pid-CHECKED, like the manager's: a daemon that exits must not delete a record that
+ * already belongs to its successor (a fast restart writes the new pid before the old process has
+ * finished unwinding), so the record goes only while it still names this process.
+ *
+ * A root with no `.cotal/` is not a workstation and gets no record — a hosted composition has none,
+ * and creating one here would plant a stray workspace root wherever the daemon happened to run.
+ */
+export function recordDeliveryPid(root, space) {
+    if (!existsSync(join(root, ".cotal"))) {
+        console.error(`• delivery: no ${join(root, ".cotal")} here, so this daemon is not recorded in a pidfile (nothing local will find it by pid)`);
+        return () => { };
+    }
+    const ctx = { root, space };
+    reclaimDeadPreUpgradeRecord(DELIVERY_PIDFILE, ctx);
+    const pidPath = canonicalLocalProcessPath(DELIVERY_PIDFILE, ctx);
+    const mine = String(process.pid);
+    writeFileSync(pidPath, mine);
+    // #969: pin the pid to its process start so a later teardown can refuse a reused pid.
+    writeIdentityPin(pidPath, process.pid);
+    return () => {
+        try {
+            if (readFileSync(pidPath, "utf8").trim() !== mine)
+                return; // a successor's record: not ours to remove
+            removeIdentityPin(pidPath);
+            rmSync(pidPath, { force: true });
+        }
+        catch {
+            /* already gone, or unreadable: leaving a record we cannot prove is ours is the safe error */
+        }
+    };
+}
 /**
  * Run the delivery daemon: the server-side Plane-3 durable backstop. A thin composition root that
  * builds a scoped `delivery` endpoint, acquires the single-flight lease, and runs the existing
@@ -244,6 +292,35 @@ async function loadDeliveryCreds(src, v) {
  * the manager and this daemon are handed the SAME store.
  */
 export async function runDelivery(args, store) {
+    // A START-UP FAILURE AFTER THE ACQUIRE MUST GIVE THE SHARD BACK, INCLUDING WHEN THE CALLER
+    // CATCHES IT. `runDelivery` is awaited inside the CLI dispatcher's own try/catch, so a rejection
+    // out of start-up is an ordinary handled error: the process prints one line and exits 1 while the
+    // lease row still claims the shard for the rest of the 30s bucket TTL, and the next `cotal up` is
+    // refused. That is this issue's outage reached without any signal, starvation or broker fault.
+    //
+    // The release therefore hangs off the REJECTION rather than off a process event. `runStartedDelivery`
+    // publishes its own releaser once the shard is actually ours, so before the acquire there is nothing
+    // to give back and this catch re-throws untouched.
+    //
+    // THE FAULT STAYS A FAULT. This never swallows: the original error is re-thrown after the release,
+    // so the CLI prints the same line and exits with the same non-zero code it would have without any
+    // of this. Cleaning up a lease must not turn a start-up failure into a quiet one.
+    let releaseOnStartupFailure;
+    try {
+        await runStartedDelivery(args, store, (r) => { releaseOnStartupFailure = r; });
+    }
+    catch (e) {
+        if (releaseOnStartupFailure !== undefined) {
+            console.error(`\u2717 delivery: start-up failed - releasing the shard before exiting`);
+            try {
+                await releaseOnStartupFailure();
+            }
+            catch { /* the throw below is the report */ }
+        }
+        throw e;
+    }
+}
+async function runStartedDelivery(args, store, publishReleaser) {
     const v = args.values;
     const shard = v.shard ? Number(v.shard) : 0;
     const shards = v.shards ? Number(v.shards) : 1;
@@ -302,6 +379,11 @@ export async function runDelivery(args, store) {
     await validateScanTargetAdmission(scanTarget);
     console.error(`• delivery: $SYS sweeps bound to ${join(scanTarget.root, ".cotal")} (account ${scanTarget.expectedAccount})`);
     const reloadStoreIdentity = reloadStoreIdentityOf(credsSrc);
+    // THIS daemon's connection nkey, pinned once. It is what the endpoint AUTHENTICATES as, and it is
+    // what `card.id` must carry at construction (a creds SOURCE has no cred to derive it from yet).
+    // It is NOT what the lease record carries as `holder`, see readOwnLease, which compares against
+    // `ep.card.id`, the value the endpoint actually stamps.
+    const ownId = idFromCreds(creds.initial);
     const ep = new CotalEndpoint({
         space,
         servers: server,
@@ -314,7 +396,7 @@ export async function runDelivery(args, store) {
         consume: false, // it pulls the Plane-3 consumers itself; no agent live-tail
         watchPresence: true, // read the roster for @mention resolution …
         registerPresence: false, // … but NEVER publish the daemon onto the roster (it's infra, not a peer)
-        card: { id: idFromCreds(creds.initial), name: "delivery", role: "delivery", kind: "endpoint" },
+        card: { id: ownId, name: "delivery", role: "delivery", kind: "endpoint" },
     });
     // Both channels: raw connection errors ride `error`, while every condition the endpoint is
     // already surviving — a failed 75% renewal, the passive backstop's "still holds the previous
@@ -327,6 +409,11 @@ export async function runDelivery(args, store) {
     // Acquire the single-flight lease BEFORE binding the loops: a loud refusal-to-bind if another daemon
     // already holds this shard (two clients binding the same durable name SPLIT delivery). The bucket TTL
     // frees a crashed holder's lease so a fresh daemon re-acquires.
+    // THE REVISION THIS PROCESS OWNS, or `undefined` once it knows it does not own the row any more.
+    // `releaseDeliveryLease` takes it as a compare-and-swap, so a shutdown that no longer holds the
+    // shard releases NOTHING rather than deleting whatever row is there. The exits where that matters
+    // are exactly the takeover exits below: without it, the departing daemon removes the REPLACEMENT's
+    // lease on its way out and leaves the shard with no holder at all.
     let revision;
     try {
         revision = await ep.acquireDeliveryLease(shard);
@@ -337,6 +424,125 @@ export async function runDelivery(args, store) {
         process.exit(1);
         return;
     }
+    // SIGNAL HANDLING IS ARMED HERE, THE STATEMENT AFTER THE SHARD BECOMES OURS, not at the end of
+    // start-up, and not merely before the readiness flip. From this line on there is a row on the
+    // broker with this daemon's name on it, and every instant until a handler exists is an instant in
+    // which SIGTERM takes Node's DEFAULT action: immediate death, no release, the shard claimed by a
+    // process that no longer exists for the rest of the 30s bucket TTL. The next `cotal up` is then
+    // refused outright with "a live lease already exists".
+    //
+    // THE WINDOW WAS REACHABLE FROM THE PUBLIC CLI, and reviewers reproduced it deterministically:
+    // SIGTERM at the instant the lease row turns ready killed the daemon 6/6 times with the row left
+    // behind and a real replacement refused, while the same signal 1500ms later released cleanly 2/2.
+    // Registration used to sit ~460 lines below, past the readiness flip and the awaited membership
+    // and timer-writer starts, so `cotal up` could return on a ready row while the daemon was still
+    // defenceless. Moving it merely below `shutdown`'s own definition is NOT enough: that still sits
+    // after markReady and after `startMembership`.
+    //
+    // The handler cannot call `shutdown` (defined far below, over state that does not exist yet), so
+    // it does the one thing that is always correct here and never wrong later: give the lease back if
+    // it is provably ours, then exit. Once the real `shutdown` is installed it REPLACES this, and
+    // `stopping` makes the pair idempotent, so a signal arriving mid-swap cannot run both.
+    let stopping = false;
+    /** Removes THIS process's liveness record, once it has one. Declared BEFORE the handlers that
+     *  call it and assigned below, so a signal landing between the two is a no-op rather than a
+     *  temporal-dead-zone throw: at that instant nothing has been written, so there is nothing to
+     *  remove. Every exit path from the acquire onward goes through one of the three closures that
+     *  call it, which is what keeps the record's lifetime equal to this daemon's. */
+    let unrecordPid;
+    const earlyStop = (code) => {
+        if (stopping)
+            return;
+        stopping = true;
+        setTimeout(() => process.exit(code), 2000);
+        unrecordPid?.();
+        void (async () => {
+            try {
+                const own = await ep.readDeliveryLeaseEntry(shard);
+                if (own !== undefined && ep.ownsDeliveryLease(own.info))
+                    await ep.releaseDeliveryLease(shard, own.revision);
+            }
+            catch { /* broker may be gone, the bucket TTL is the crash-safe release authority */ }
+            try {
+                await ep.stop();
+            }
+            catch { /* broker may be gone */ }
+            process.exit(code);
+        })();
+    };
+    const earlySigint = () => earlyStop(0);
+    const earlySigterm = () => earlyStop(0);
+    process.on("SIGINT", earlySigint);
+    process.on("SIGTERM", earlySigterm);
+    // THE LIVENESS RECORD GOES HERE, AFTER THE ACQUIRE AND NOT BEFORE IT (#1528).
+    //
+    // The record answers "which process is this space's delivery daemon", and until this line this
+    // process cannot answer it: the lease is the single-flight admission, and a daemon that loses the
+    // CAS above REFUSES TO BIND and exits. Writing on entry would let such a loser overwrite the live
+    // holder's record on its way out the door — a fresh record naming a process that is about to be
+    // gone, which is the same lie this issue is about with a newer timestamp on it. The acquire is
+    // the first instant the answer is this process, so it is the first instant the record may say so.
+    //
+    // It is also written BEFORE `startPlane3`, deliberately: from the acquire onward there is a row on
+    // the broker with this daemon's name on it, so an operator's `cotal down` must be able to FIND
+    // this process even if the bind below hangs or fails. Readiness is a separate fact and the lease's
+    // own `ready` flag already carries it; the pidfile only ever claimed a process.
+    //
+    // Placed one statement after the signal handlers, so every exit path from here on can remove it.
+    unrecordPid = recordDeliveryPid(findCotalRoot(), space);
+    // AND THE SAME RELEASE, REACHABLE BY AN ORDINARY `catch`. The two process-level guards below
+    // only see a fault that reaches the RUNTIME. A start-up rejection on the public CLI path does
+    // not: `runCli` awaits this function inside its own try/catch (cli/src/command.ts), so the
+    // rejection is HANDLED, `unhandledRejection` never fires, and the process exits 1 through the
+    // CLI's own error line with the shard still claimed. A reviewer reproduced exactly that against
+    // a real broker by pre-creating a conflicting fan-out durable so `startPlane3` rejects after the
+    // acquire: exit 1, row still on the broker, replacement refused with "a live lease already
+    // exists". So the release is published HERE, to a scope that a plain `catch` around the rest of
+    // start-up can reach, and the process guards stay as the backstop for faults that never become
+    // a rejection this function can see (a synchronous throw in a timer, say).
+    publishReleaser(async () => {
+        if (stopping)
+            return;
+        stopping = true;
+        unrecordPid?.(); // the record dies with the daemon it describes, including on a start-up failure
+        try {
+            const own = await ep.readDeliveryLeaseEntry(shard);
+            if (own !== undefined && ep.ownsDeliveryLease(own.info))
+                await ep.releaseDeliveryLease(shard, own.revision);
+        }
+        catch { /* broker may be gone - the bucket TTL is the crash-safe release authority */ }
+        try {
+            await ep.stop();
+        }
+        catch { /* broker may be gone */ }
+    });
+    // A START-UP FAILURE AFTER THE ACQUIRE MUST ALSO GIVE THE SHARD BACK. Between this point and the
+    // handler swap far below, a throw would otherwise propagate out of `runDelivery` with the row
+    // still claiming the shard, stranding it for the bucket TTL exactly as an unhandled signal did -
+    // a different door into the same outage. So the release happens HERE, where the failure is,
+    // rather than being trusted to a caller that may not have one.
+    //
+    // THE FAULT STAYS LOUD, AND STAYS A FAULT. These guards change WHEN the process dies, never
+    // whether it reports why: the original error is printed with its stack, as Node's default handler
+    // would, and the exit code is non-zero. Cleaning up a lease must not turn a crash into a quiet
+    // stop - that would trade this issue's outage for a silent one.
+    //
+    // DISARMED BEFORE THEY CAN RECURSE. `earlyStop`'s own async body can itself reject (the broker may
+    // be mid-teardown), which would re-enter these listeners while cleanup is in flight. They are
+    // removed on first use, so a cleanup fault falls through to the default handler and the 2s
+    // hard-exit timer remains the backstop. `stopping` already makes a second entry a no-op; this
+    // makes the recursion impossible rather than merely harmless.
+    const earlyFault = (what, err) => {
+        process.off("uncaughtException", earlyUncaught);
+        process.off("unhandledRejection", earlyRejection);
+        console.error(`\u2717 delivery: start-up ${what} - releasing shard ${shard} before exiting`);
+        console.error(err instanceof Error ? (err.stack ?? err.message) : String(err));
+        earlyStop(1);
+    };
+    const earlyUncaught = (e) => earlyFault("faulted", e);
+    const earlyRejection = (e) => earlyFault("rejected a promise", e);
+    process.on("uncaughtException", earlyUncaught);
+    process.on("unhandledRejection", earlyRejection);
     // Broker-sourced graph membership handle — declared BEFORE Plane-3 so the delivery-admin reload
     // hook below can close over it (it starts further down; the closure reads it live).
     let membership;
@@ -398,7 +604,6 @@ export async function runDelivery(args, store) {
         membershipDown = e.message;
         console.error(`! membership: failed to start (${membershipDown}); graph membership degraded, delivery unaffected`);
     }
-    let stopping = false;
     // The TIMER WRITER (SPEC 13.2): the pump that turns workflow `.schedule` requests into armed
     // broker schedules. Hosted here because this daemon is the space's standing server-side process;
     // without a running writer no pause on the space ever expires. Its OWN connection under the same
@@ -447,10 +652,54 @@ export async function runDelivery(args, store) {
         stopping = true;
         clearInterval(renew);
         clearInterval(brokerWatch);
+        // THE RECORD GOES FIRST, AND SYNCHRONOUSLY. Everything below this line talks to a broker that
+        // may be dead and is bounded only by the 2s hard exit; a record left behind because a drain
+        // hung is a record that outlives its process, which is this issue's defect re-entering through
+        // the exit path. `rmSync` needs no broker and cannot hang, so it runs before any of it.
+        unrecordPid?.();
         // Hard-exit fallback: a graceful release/stop talks to the broker, which may be DEAD (the broker-gone
         // exit path) — don't let that hang the process. Force exit if the graceful path doesn't finish quickly.
         setTimeout(() => process.exit(code), 2000);
         void (async () => {
+            // THE LEASE GOES FIRST. Everything else here is this process's own teardown, which nobody is
+            // waiting on; the lease row is the one piece of SHARED state, and while it survives no
+            // replacement daemon can acquire the shard at all (the acquire is an atomic create, so it is
+            // refused outright and the row blocks the slot for the rest of the 30s bucket TTL). Behind
+            // three awaits it was reachable only if all three finished inside the 2s hard-exit above, and
+            // membership.stop() and the timer writer's stop+drain each talk to the broker on their own
+            // connections.
+            //
+            // RELEASE AGAINST THE BROKER'S REVISION, NOT THE ONE THIS PROCESS HAPPENS TO HOLD.
+            //
+            // The release is a compare-and-swap, so a token one step behind frees NOTHING, and it fails
+            // SILENTLY: `releaseDeliveryLease` swallows every error by design, because at shutdown the
+            // broker may already be gone. A daemon can therefore stop cleanly, believe it released, and
+            // leave its row claiming the shard for the rest of the 30s bucket TTL - after which the next
+            // `cotal up` is refused with "a live lease already exists" and the shard is unservable by
+            // anyone. That is this issue's outage reached without any starvation or broker fault at all.
+            //
+            // THE CACHED TOKEN CAN LAG THE BROKER, and it does not take a fault to get there. The claim is
+            // deliberately that general, because more than one ordinary interleaving produces it and the
+            // trace does not distinguish them: `markDeliveryLeaseReady` MOVES the row, so between the
+            // broker applying that write and the daemon assigning the returned value, `revision` still
+            // holds the acquire token - and a shutdown running in that interval argues it. A markReady
+            // that rejects AFTER its write landed leaves the same lag permanently, since the failure is
+            // caught (correctly: the renew loop repairs it). The launcher can already have observed the
+            // row ready in either case, so from outside this is a started daemon. Measured, with both
+            // controls in one run: release at the acquire revision left the row in place, release at the
+            // current revision removed it, and a replacement endpoint could then acquire.
+            //
+            // So: re-read, and release only what is PROVABLY ours. `ownsDeliveryLease` is the same test
+            // the renew path uses, so a row belonging to a successor is left alone - which is the property
+            // the CAS was protecting in the first place, now argued from evidence rather than from a
+            // token that may be out of date. No row, or not ours: nothing to release, and the TTL remains
+            // the crash-safe authority for anything this path cannot reach.
+            try {
+                const own = await ep.readDeliveryLeaseEntry(shard);
+                if (own !== undefined && ep.ownsDeliveryLease(own.info))
+                    await ep.releaseDeliveryLease(shard, own.revision);
+            }
+            catch { /* broker may be gone - the bucket TTL is the crash-safe release authority */ }
             try {
                 await membership?.stop();
             }
@@ -463,10 +712,6 @@ export async function runDelivery(args, store) {
                 await timerWriter?.nc.drain();
             }
             catch { /* broker may be gone */ }
-            try {
-                await ep.releaseDeliveryLease(shard);
-            }
-            catch { /* broker may be gone */ }
             try {
                 await ep.stop();
             }
@@ -474,25 +719,325 @@ export async function runDelivery(args, store) {
             process.exit(code);
         })();
     };
-    // Renew the lease at ~half the TTL so a healthy holder never self-evicts; losing the CAS means
-    // another daemon took over (we exit rather than double-deliver).
+    // SIGNAL HANDLERS GO UP THE MOMENT `shutdown` EXISTS, NOT AFTER STARTUP FINISHES.
+    //
+    // The lease is acquired long before this point, and registration used to sit at the very end of
+    // start-up, past 22 further `await`s (Plane-3 bind, membership feed, timer writer, broker watch).
+    // A SIGTERM landing in that window hit Node's DEFAULT handler and killed the process outright:
+    // no release, no CAS, the row left behind for the rest of the 30s bucket TTL. The daemon had
+    // already announced itself up, so from outside it was a fully started daemon that died silently
+    // holding the shard, and the next `cotal up` was refused with "a live lease already exists", the
+    // shard unservable for 30s after a perfectly ordinary stop-and-restart.
+    //
+    // Measured, not reasoned: with a diagnostic on the handler, the failing run's daemon log shows
+    // the up line and NO `SIGTERM received`, while the two roots that passed in the same run show it.
+    // That is what reddened `delivery-refresh-keeps-tls`, which stops the daemon and relaunches at
+    // once. Registering here closes the window to the span before the lease exists, where there is
+    // nothing to release.
+    //
+    // HAND OVER FROM `earlyStop`. The early handlers armed at the lease acquire are REMOVED rather
+    // than left alongside these: `process.on` appends, so both would fire and the early one - which
+    // knows nothing of the renew timer, the membership feed or the timer writer - would run first and
+    // set `stopping`, making the full teardown a no-op. `stopping` still guards the swap itself, so a
+    // signal delivered between these two statements is handled exactly once.
+    process.off("SIGINT", earlySigint);
+    process.off("SIGTERM", earlySigterm);
+    // The start-up fault guards go too, and BY REFERENCE: `removeAllListeners` here would strip
+    // listeners this daemon does not own. They exist to cover the window where no `shutdown` exists;
+    // past this line a fault should surface normally rather than become a quiet exit that skips the
+    // full teardown.
+    process.off("uncaughtException", earlyUncaught);
+    process.off("unhandledRejection", earlyRejection);
+    process.on("SIGINT", () => shutdown(0));
+    process.on("SIGTERM", () => shutdown(0));
+    /** What the broker says about THIS shard's lease key right now, the verdict a failed renew does
+     *  NOT have. `unknown` never collapses into `gone`: not being able to look is not the same fact as
+     *  looking and finding nothing (#1318).
+     *
+     *  The ownership test is `ep.ownsDeliveryLease`, which compares the row against BOTH this
+     *  endpoint's wire identity and its per-run incarnation. Neither half is optional, and the two
+     *  failures they rule out are different:
+     *
+     *  1. Comparing against `ownId` (the bare connection nkey `U…`) is comparing across NAMESPACES:
+     *     the endpoint rewrites `card.id` in its constructor to the wire PRINCIPAL dot-form
+     *     `${owner}.${actor}` (`local.U…`), and that is what `encodeLease` stamps. That mismatch made
+     *     `held` UNREACHABLE, the daemon read its own row, failed to recognise itself, took the
+     *     `taken` branch and exited naming ITSELF as the thief, leaving a not-ready row and no
+     *     process. `held` is the one survivable reading ("your renew failed but the shard is still
+     *     yours"), so making it unreachable turned every recoverable renew failure into a permanent
+     *     withdrawal: #1318's own outage re-entering through the path built to prevent it.
+     *
+     *  2. Comparing on the principal ALONE is not sufficient either, and this is the one that looks
+     *     correct: the daemon's cred is a FILE that every restart re-reads, so a REPLACEMENT daemon
+     *     authenticates as the same nkey and writes the same `holder`. A displaced daemon would then
+     *     read its successor's row, conclude the shard was still its own, carry on serving a shard it
+     *     had lost (two daemons on one durable, which is the split this lease exists to prevent) and
+     *     CAS-release the live holder's row on the way out. Cells F and G stage exactly that, with two
+     *     daemons sharing one creds file, because that is what the product does. */
+    const readOwnLease = async () => {
+        let current;
+        try {
+            current = await ep.readDeliveryLeaseEntry(shard);
+        }
+        catch (e) {
+            return { kind: "unknown", why: e.message };
+        }
+        if (current === undefined)
+            return { kind: "gone" };
+        if (!ep.ownsDeliveryLease(current.info))
+            return { kind: "taken", by: current.info.holder };
+        // Held by us, AND at the broker's own revision rather than the one this process last cached.
+        // That distinction is load-bearing: the renew that just failed may have been applied before its
+        // reply was lost, in which case the cached revision is permanently one behind and every later
+        // CAS is refused over a sequence this daemon moved itself. Adopting the read revision is what
+        // lets a survivable renew failure actually be survived. (An earlier comment here claimed the
+        // record carries no revision; it does, the KV entry's own `revision`, and that claim was
+        // wrong, which is why the stale token went unnoticed.)
+        return { kind: "held", revision: current.revision };
+    };
+    /** Print a lease state only when it CHANGES. The renew ticks every few seconds, so a broker
+     *  outage would otherwise write the same line thousands of times into the daemon's log; an
+     *  operator needs the line going in and the line coming out. */
+    let leaseState = "held";
+    const noteLease = (state, what) => {
+        if (state === leaseState)
+            return;
+        leaseState = state;
+        console.error(`${state === "held" ? "✓" : "!"} delivery: ${what} (space ${space}, shard ${shard})`);
+    };
+    /** Announce going quiet, once per quiesced episode. Same rule as {@link noteLease}: the renew ticks
+     *  forever, and an operator needs the edge, not a log line per tick. */
+    let quiesced = false;
+    const noteQuiesce = () => {
+        if (quiesced)
+            return;
+        quiesced = true;
+        console.error(`! delivery: stopped serving shard ${shard} (fan-out, reader and control unbound) while it re-checks who owns the lease (space ${space})`);
+    };
+    /** Resume serving, and CLOSE the quiesced episode so a later one announces itself again. Every
+     *  caller has just proven ownership; `why` is what proved it, since "it started serving again" is
+     *  only auditable next to the evidence that permitted it. */
+    const resumeServing = async (why) => {
+        try {
+            await ep.rearmPlane3();
+        }
+        catch (e) {
+            console.error(`! delivery: ${why} but could not resume Plane-3 (${e.message})`);
+            return;
+        }
+        // `ready` MEANS "THE RESPONDER IS UP", and it is what `ensureDelivery` waits on and what the
+        // `cotal_channels` health surface reports. Startup flips it only after binding, for exactly that
+        // reason, and a re-acquire creates the row afresh, which `acquireDeliveryLease` deliberately
+        // makes NOT-ready. Without this flip a daemon that recovered would serve correctly while every
+        // readiness waiter in the space timed out against a permanently not-ready lease: the outage
+        // #1318 is about, surviving the repair by hiding in the readiness flag instead of the exit path.
+        // Ordered AFTER the re-arm so the flag never claims more than has actually been bound.
+        if (revision !== undefined) {
+            try {
+                revision = await ep.markDeliveryLeaseReady(shard, revision);
+            }
+            catch (e) {
+                console.error(`! delivery: resumed serving but could not mark the lease ready (${e.message}); the next renew re-synchronises`);
+            }
+        }
+        if (!quiesced)
+            return;
+        quiesced = false;
+        console.error(`\u2713 delivery: serving shard ${shard} again, ${why} (space ${space})`);
+    };
+    // Renew the lease at ~half the TTL so a healthy holder never self-evicts.
+    //
+    // A FAILED RENEW IS A QUESTION, NOT A VERDICT (#1318). The shipped code exited on ANY renew
+    // error, so a key that had EXPIRED under local CPU starvation, with nobody else holding it,
+    // reported as `wrong last sequence: 0`, was indistinguishable from a genuine takeover, and the
+    // only holder there was ended itself. The verdict comes from RE-READING the key: `taken` exits so
+    // the holder stays single, `gone` is repaired by an ATOMIC create (which arbitrates: if a
+    // replacement got there first the create fails and THAT is the genuine loss), and `held`/`unknown`
+    // keep serving. Overlap-guarded: two CAS attempts against the same cached revision would have the
+    // second refused over a sequence the first legitimately moved, a conflict this daemon manufactures
+    // itself and then reads as someone else's takeover.
+    let renewInFlight = false;
     const renew = setInterval(() => {
-        ep.renewDeliveryLease(shard, revision)
-            .then((r) => (revision = r))
-            .catch((e) => {
-            console.error(`✗ delivery: lost the lease (${e.message}) — exiting so the holder is single`);
-            shutdown(1);
-        });
+        if (stopping || renewInFlight)
+            return;
+        renewInFlight = true;
+        void (async () => {
+            try {
+                // A renew with no owned revision is not a renew: this process already established that the
+                // shard is someone else's and is on its way out. Attempting one would either fail noisily or,
+                // worse, be graded as a lease question when ownership is already settled.
+                if (revision === undefined)
+                    return;
+                revision = await ep.renewDeliveryLease(shard, revision);
+                // A SUCCESSFUL CAS RENEW IS PROOF OF OWNERSHIP, so it is also a re-arm point. Without this a
+                // daemon that quiesced on `unknown` (the broker could not answer who owns the shard) would
+                // stay silent forever once the broker came back: the `held` re-arm below only runs on the
+                // failure path, and the failure path stops running as soon as renews succeed again. Quiesce
+                // must be recoverable by the same evidence that makes it unnecessary.
+                await resumeServing("it renewed its lease");
+                noteLease("held", "renews its delivery lease again");
+                return;
+            }
+            catch (renewError) {
+                const why = renewError.message;
+                // GO QUIET BEFORE ASKING. From here this process does not know whether the shard is still
+                // its own, and a CAS on the lease row keeps one ROW, not one SERVER: across the re-read and
+                // the re-acquire below, an un-quiesced daemon would still be consuming the fan-out durable,
+                // still running the reader, and still answering ctl.delivery, so a replacement that won the
+                // shard in that window would SPLIT the durable with it. A review finding, and the pre-fix
+                // code did not have this window because it simply exited on the first renew failure.
+                try {
+                    await ep.quiescePlane3();
+                    // WITHDRAW THE READINESS CLAIM TOO. `ready` asserts the responder is up; it is now down, so
+                    // leaving it set would tell every `ensureDelivery` waiter in the space to keep waiting on a
+                    // daemon that is deliberately not answering. Best-effort: the renew that just failed means
+                    // the CAS may fail too, and staying quiet matters more than the flag being tidy. The row
+                    // itself is kept, the shard is still claimed, only the answering claim is withdrawn.
+                    if (revision !== undefined) {
+                        try {
+                            revision = await ep.markDeliveryLeaseNotReady(shard, revision);
+                        }
+                        catch { /* the row may have moved on; the ownership read below is what decides */ }
+                    }
+                    // ANNOUNCED, because an operator watching a stall needs to know the daemon stopped serving
+                    // on purpose rather than silently wedged, and because the live cell anchors on this line
+                    // to know when to start demanding that this process holds no Plane-3 bindings.
+                    noteQuiesce();
+                }
+                catch (e) {
+                    console.error(`! delivery: could not quiesce Plane-3 while checking the lease (${e.message})`);
+                }
+                const reading = await readOwnLease();
+                switch (leaseAction(reading)) {
+                    case "keep-serving":
+                        // Still ours, or unanswerable. Only a PROVEN `held` re-arms: `unknown` stays quiet,
+                        // because not being able to ask is not permission to keep acting on the shard.
+                        if (mayServeOn(reading)) {
+                            // ADOPT THE BROKER'S REVISION before serving again. The failed renew may have landed
+                            // with its reply lost, so the cached token can be stale; re-arming on it would leave
+                            // every later CAS refused over this daemon's own write and manufacture the takeover
+                            // it is trying to rule out. The read just told us the sequence, use it.
+                            // Narrowed by the guard above: `mayServeOn` admits only `held`, which is the one
+                            // reading that carries a revision. Stated as an assert rather than a cast so a future
+                            // widening of `mayServeOn` fails here loudly instead of serving without a CAS token.
+                            if (reading.kind !== "held")
+                                throw new Error(`delivery: mayServeOn admitted a "${reading.kind}" reading, which carries no revision to serve on`);
+                            revision = reading.revision;
+                            await resumeServing("re-reading the key showed the lease is still its own");
+                        }
+                        noteLease(reading.kind === "held" ? "held-unrenewed" : "unknown", reading.kind === "held"
+                            ? `could not renew its lease (${why}) but the key is still its own; serving, retrying`
+                            // NOT "serving": the quiesce above is still in force on this branch, and `mayServeOn`
+                            // refuses `unknown`, so nothing re-armed. Saying "serving" here would describe the
+                            // pre-fix behaviour and hide the very state this repair introduced.
+                            : `could not renew its lease (${why}) or re-read it (${reading.kind === "unknown" ? reading.why : ""}); staying quiet, retrying until the broker answers`);
+                        return;
+                    case "reacquire":
+                        try {
+                            revision = await ep.acquireDeliveryLease(shard);
+                            // Won the atomic create, so the shard is provably ours again: resume serving.
+                            await resumeServing(`it won the atomic create at revision ${revision}`);
+                            noteLease("held", `found its lease key gone (renew: ${why}) and re-acquired it at revision ${revision}`);
+                        }
+                        catch (e) {
+                            // Refused: a live lease exists that is not ours, so a replacement daemon holds this
+                            // shard. THIS is the genuine loss, and it exits, WITHOUT a revision to release, or
+                            // the exit would delete the replacement's row.
+                            revision = undefined;
+                            console.error(`✗ delivery: lost the lease (${why}) and another daemon has taken shard ${shard} (${e.message}), exiting so the holder is single`);
+                            shutdown(1);
+                        }
+                        return;
+                    case "exit":
+                        // Another daemon's row. Forget our revision first: releasing on the way out would delete
+                        // the holder's lease and turn a clean handover into an unheld shard.
+                        revision = undefined;
+                        console.error(`✗ delivery: the lease for shard ${shard} is held by ${reading.kind === "taken" ? reading.by : "another daemon"}, not by this process (renew: ${why}), exiting so the holder is single`);
+                        shutdown(1);
+                        return;
+                }
+            }
+            finally {
+                renewInFlight = false;
+            }
+        })();
     }, Math.max(1000, Math.floor(LEASE_TTL_MS / 2)));
     // Coupled to the broker: POLL its reachability. Survive brief blips (the endpoint reconnects on its
-    // own), but EXIT if the broker has been gone for BROKER_GONE_MS — the endpoint would otherwise retry
-    // reconnect forever (its terminal-close never fires), so this is what stops the daemon outliving the
-    // server it serves. (`cotal up`/`down` teardown stops it too.) The window is env-overridable for tests.
+    // own), but EXIT if the broker is GONE, the endpoint would otherwise retry reconnect forever (its
+    // terminal-close never fires), so this is what stops the daemon outliving the server it serves.
+    // (`cotal up`/`down` teardown stops it too.) The window is env-overridable for tests.
+    //
+    // WHAT "GONE" MEANS IS NOW DECIDED FROM EVIDENCE, NOT FROM A CLOCK (#1318). Three conditions used
+    // to produce one signal and only one of them was a dead server: the broker being down, this
+    // process being descheduled so the interval never fired, and a probe that could not complete a
+    // handshake because the local process could not get scheduled to finish it. A wall-clock
+    // `Date.now() - lastReachable` cannot tell them apart, and widening it only moves the threshold.
+    // Three signals separate them, and all three are things this process can actually observe:
+    //
+    //   • MEASURED LOOP LAG. The gap between consecutive firings of a timer we own, minus its nominal
+    //     period, is local starvation by direct measurement. It is credited back, so time this process
+    //     spent off the runqueue is never counted against the broker.
+    //   • COMPLETED NEGATIVE PROBES. Only a probe that RAN TO COMPLETION and returned false is
+    //     evidence about the server. A probe that never ran contributes nothing (that was the whole
+    //     defect: the window aged with no probe having failed), and one that REJECTED is an
+    //     unanswered question, it now resets the counter and logs, where it used to be swallowed by
+    //     `.catch(() => {})` and silently age the window.
+    //   • TRANSPORT-LEVEL LIVENESS. A `transport: connected` edge from the endpoint's own connection
+    //     cannot happen without a server on the other end, so it is positive evidence obtained for
+    //     free, on a path that does not need this process to schedule a probe at all.
+    //
+    // A starvation diagnosis therefore reports DEGRADED and keeps serving; a genuinely dead broker
+    // still exits, on the same window, as fast as completed probes can say so.
     const BROKER_GONE_MS = Number(process.env.COTAL_DELIVERY_BROKER_GONE_MS) || 15_000;
+    // How many completed negatives make elapsed time believable. Two is the floor: one completed
+    // negative is a single refused connect, which a loopback under momentary pressure can produce.
+    // The default scales with the window so a test that shortens the window does not thereby demand
+    // more evidence than the window has room for.
+    const BROKER_GONE_PROBES = Math.max(2, Number(process.env.COTAL_DELIVERY_BROKER_GONE_PROBES) || Math.ceil(BROKER_GONE_MS / PROBE_INTERVAL_MS / 2));
+    // The hard backstop: past this, no amount of measured lag or transport optimism keeps the daemon
+    // alive. A daemon that OUTLIVES a dead broker is worse than one that restarts unnecessarily, so
+    // the repair is bounded and fails toward exiting.
+    const BROKER_GONE_BACKSTOP_MS = Math.max(BROKER_GONE_MS, Number(process.env.COTAL_DELIVERY_BROKER_GONE_BACKSTOP_MS) || BROKER_GONE_MS * 4);
     let lastReachable = Date.now();
+    let completedNegatives = 0;
+    let degraded = false;
+    // What the endpoint's OWN connection reports about its socket to this same broker. Seeded true
+    // because `ep.start()` above completed, which it cannot do without a server having answered.
+    let transportConnected = true;
+    const lag = new LoopLagMeter(PROBE_INTERVAL_MS);
+    /** Positive evidence, from wherever it came: restart the window and its lag budget together. */
+    const sawBroker = () => {
+        lastReachable = Date.now();
+        completedNegatives = 0;
+        lag.reset();
+        if (degraded) {
+            degraded = false;
+            console.error(`✓ delivery: the broker is answering again, Plane-3 is serving normally (space ${space})`);
+        }
+    };
+    // The endpoint's OWN transport edge. This is evidence the daemon gets without being scheduled to
+    // probe for it, and it is the signal that distinguishes "the connection object reports a
+    // transport-level close" from "silence": a live connection to that address proves a server, while
+    // a disconnect is the endpoint's own business (nats.js reconnects through blips of its own
+    // accord) and merely stops excusing a probe that will not complete.
+    ep.on("transport", (t) => {
+        transportConnected = t.connected;
+        if (t.connected)
+            sawBroker();
+    });
+    // THE BUDGET THE PROBE IS ACTUALLY GIVEN, asked of the one function that decides it. A ws(s)
+    // broker rides an HTTPS edge and gets 5s, not the 1s a loopback TCP broker gets; judging a ws
+    // probe against 1000 would read every honest refusal on such a broker as this process's own
+    // starvation, so the completed-negative count could never rise and a genuinely dead ws broker
+    // would be ended only by the backstop, with the wrong reason in its log. The budget and the
+    // judgment have to come from the same place.
+    const probeBudgetMs = defaultProbeTimeoutMs(server);
     const brokerWatch = setInterval(() => {
         if (stopping)
             return;
+        // Measure FIRST, before any await: this is the gap since the previous firing, and it is the
+        // only place the daemon can learn that it was not scheduled.
+        lag.tick(Date.now());
         // THE SAME TRANSPORT AS EVERY OTHER DIAL IN THIS PROCESS. This poll carries `latestCreds` — a
         // standing credential — and `isReachable` performs a real authenticated connect whenever creds
         // are supplied, every 2 seconds, for the life of the daemon. It is the most repeated credential
@@ -502,21 +1047,102 @@ export async function runDelivery(args, store) {
         // least consistent. What is new is the ASYMMETRY: with the two dials above upgraded, an operator
         // who enables TLS would get a protected main path and an unprotected watchdog. An inconsistent
         // guarantee is worse than a uniformly absent one, because the operator now believes something.
+        const probeStarted = Date.now();
+        // Watch this process's own scheduling FOR THE DURATION OF THE PROBE. A refusal is only evidence
+        // about the server if the server was actually given the time the deadline promised it, and
+        // under short CPU slices most of a probe's wall-clock can be time this process was not running.
+        const sampler = new DescheduleSampler();
+        sampler.start(probeStarted);
         void isReachable(server, { creds: latestCreds, ...(tls ? { tls: true } : {}) })
-            .then((ok) => {
-            if (ok) {
-                lastReachable = Date.now();
+            .then((ok) => classifyProbe(ok, Date.now() - probeStarted, probeBudgetMs, PROBE_LATE_FACTOR, sampler.stop()), 
+        // A REJECTED probe is an unanswered question, not a negative answer. It used to be swallowed
+        // whole by `.catch(() => {})`, so it neither refreshed the window nor evaluated anything and
+        // silently aged the daemon toward an exit it had gathered no evidence for.
+        (e) => {
+            sampler.stop();
+            console.error(`! delivery: the broker probe did not complete (${e.message}), no verdict from it; serving, retrying`);
+            return classifyProbe(undefined, Date.now() - probeStarted);
+        })
+            .then((probe) => {
+            if (stopping)
+                return;
+            if (probe.counts === "positive") {
+                sawBroker();
+                return;
+            }
+            if (probe.counts === "incomplete") {
+                // The run of credible refusals is broken by a probe that did not complete.
+                completedNegatives = 0;
                 return;
             }
-            if (Date.now() - lastReachable > BROKER_GONE_MS) {
-                console.error(`✗ delivery: broker unreachable for >${BROKER_GONE_MS / 1000}s — exiting (coupled to the broker)`);
+            if (probe.counts === "starved") {
+                // The probe RAN and said no, but its answer arrived so far past its own deadline that the
+                // deadline was enforced against this process rather than against the server. That is the
+                // starved-client case, and it is the one an elapsed-time predicate cannot see at all: the
+                // timer is firing, the probes are completing, and every one of them is `false`.
+                // Dated with the instant the answer ARRIVED, so the meter can tell whether this stall is
+                // the same wall-clock interval the tick above already charged (union, counted once) or an
+                // adjacent one (disjoint, both counted). The overlap question is settled by timestamps
+                // rather than by comparing magnitudes, which cannot distinguish the two.
+                lag.credit(probe.lateBy, Date.now());
+                completedNegatives = 0;
+            }
+            else {
+                // A refusal that arrived on time. This is the only thing that may accrue against the broker.
+                completedNegatives += 1;
+            }
+            // ONE SPAN, MEASURED ONCE, USED FOR BOTH. Reading the clock twice would let the elapsed span
+            // and the lag clamp disagree by the cost of the call itself.
+            const sinceReachable = Date.now() - lastReachable;
+            const verdict = brokerGoneVerdict({
+                msSinceLastReachable: sinceReachable,
+                // CLAMPED TO THE SPAN IT IS SUBTRACTED FROM. The interval gap and a late probe answer can
+                // testify to the same stall, and the meter cannot tell that from two adjacent stalls, so it
+                // keeps both charges and the bound is applied here, where the span is known. Without it a
+                // 10s stall read 17s and drove unstarved time negative, which cannot be cleared by any
+                // amount of real outage: a dead broker then survives the evidence clause and exits only on
+                // the backstop, 44s -> 62s measured. Time not had cannot exceed time passed.
+                starvedMs: lag.starvedMsWithin(sinceReachable),
+                completedNegatives,
+                transportConnected,
+                windowMs: BROKER_GONE_MS,
+                requiredNegatives: BROKER_GONE_PROBES,
+                backstopMs: BROKER_GONE_BACKSTOP_MS,
+            });
+            if (verdict.exit) {
+                // SAY WHICH EXIT THIS IS. The single message here previously claimed completed probes over
+                // ">15s of unstarved time" on BOTH paths, and on the backstop path that sentence is simply
+                // false: the backstop fires precisely when the evidence clauses did NOT conclude, often
+                // with zero completed refusals. An operator debugging a starved host was handed a
+                // confident evidentiary claim the daemon had not established.
+                console.error(verdict.reason === "backstop"
+                    ? `✗ delivery: giving up on elapsed time alone, ${Math.round(BROKER_GONE_BACKSTOP_MS / 1000)}s since the last ` +
+                        `confirmed reachability with no sufficient evidence either way (${completedNegatives} completed probes refused, ` +
+                        `${Math.round(lag.starvedMsWithin(sinceReachable) / 1000)}s of local scheduler lag credited). This is a BOUND, not a diagnosis: the ` +
+                        `broker may be gone or this process may have been starved past the bound, exiting (coupled to the broker)`
+                    : `✗ delivery: broker unreachable, ${completedNegatives} completed probes refused within their deadline over ` +
+                        `>${BROKER_GONE_MS / 1000}s of unstarved time (${Math.round(lag.starvedMsWithin(sinceReachable) / 1000)}s of local scheduler lag credited), exiting (coupled to the broker)`);
                 shutdown(1);
+                return;
+            }
+            // NOT an exit. Say so once per episode, naming WHICH condition it is, so an operator reading
+            // the log during a load incident sees "this host starved me" rather than a daemon that
+            // silently vanished.
+            if (!degraded && verdict.reason !== "reachable") {
+                degraded = true;
+                console.error(verdict.reason === "starved"
+                    ? `! delivery: DEGRADED, cannot reach the broker, but ${Math.round(lag.starvedMsWithin(sinceReachable) / 1000)}s of that window was local scheduler lag (this host is starving this process, not the broker); serving, retrying`
+                    : verdict.reason === "transport-live"
+                        ? `! delivery: DEGRADED, a fresh probe cannot complete, but this daemon's own connection to ${server} is still open, so the broker is there and this process cannot ask; serving, retrying`
+                        : `! delivery: DEGRADED, the broker has not answered for >${BROKER_GONE_MS / 1000}s but only ${completedNegatives} of ${BROKER_GONE_PROBES} probes have refused within their deadline; serving, retrying`);
             }
         })
-            .catch(() => { });
-    }, 2000);
-    process.on("SIGINT", () => shutdown(0));
-    process.on("SIGTERM", () => shutdown(0));
+            .catch((e) => {
+            // The verdict path itself faulted. Never silent, and never an exit: a bug in the detector
+            // must not end the daemon it is meant to keep alive.
+            console.error(`! delivery: the broker watch tick faulted (${e.message}); serving, retrying`);
+        });
+    }, PROBE_INTERVAL_MS);
     await new Promise(() => { }); // run until signalled
 }
 //# sourceMappingURL=delivery.js.map