@cotal-ai/core 0.49.0 → 0.50.0

package/dist/endpoint.js

@@ -23,7 +23,7 @@ import { openMembersRegistry, commitMember, tombstoneMember, activateMember, rea
 import { openAclRegistry, readAcl, readAclForAlias, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
 import { openDeliveryRegistry } from "./lease.js";
 import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, effectiveDeliveryClass, readChannelConfig, readChannelDefaults, } from "./channels.js";
-import { anycastSubject, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, controlServiceSubject, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, leaseKey, managerBucket, MANAGER_LEASE_KEY, managerLeaseKey, chatWildcard, assertValidChannel, channelInAllow, isConcreteChannel, normalizeMentions, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
+import { anycastSubject, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, controlServiceSubject, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, leaseKey, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, chatWildcard, assertValidChannel, channelInAllow, isConcreteChannel, normalizeMentions, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
 export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
 const PLANE3_FRAME_HEADER = "Cotal-Delivery-Frame";
 /** Space joined when none is given on the CLI (the `cotal-<space>` cmux tab, etc.). */
@@ -147,6 +147,10 @@ export class CotalEndpoint extends EventEmitter {
     aclKv;
     deliveryKv;
     managerLeaseKv;
+    /** Our revision of the per-space daemon-credential renewal lease (#1634), or undefined when we do
+     *  not hold it. Cleared with the bound KV handle: a revision from a dead connection is not a lease
+     *  we can prove we still hold. */
+    daemonRenewalLeaseRevision;
     membershipFeedKv;
     /** Caller-owned membership watches survive a connection rebuild as INTENT. Their iterators are
      *  connection-scoped and are stopped/re-created around the epoch swap. */
@@ -158,6 +162,9 @@ export class CotalEndpoint extends EventEmitter {
     /** When set, this endpoint hosts the Plane-3 fan-out writer + trusted reader (the server-side delivery
      *  daemon). `aclFor` maps an owner id to its current read ACL (`allowSubscribe`) for the reader's
      *  re-authorization — read FRESH per entry from the durable ACL registry KV, hence async. */
+    /** True once {@link quiescePlane3} has stopped serving this shard pending an ownership answer.
+     *  Guards {@link armPlane3} so a RECONNECT cannot silently resume serving mid-question. */
+    plane3Quiesced = false;
     plane3;
     /** Live local cache of the channel registry (key = channel token), kept by a KV watch. */
     channelConfigs = new Map();
@@ -476,8 +483,44 @@ export class CotalEndpoint extends EventEmitter {
             const claims = decodeBearerPrincipal(bearer);
             if (claims.owner !== this.owner || claims.actor !== this.actor)
                 throw new Error(`bearer source returned principal ${claims.owner}.${claims.actor}, expected ${this.owner}.${this.actor}`);
+            // Two refusals, answering different questions, and both of them are about protecting what is
+            // ALREADY held. So both are scoped to there BEING something held, which is the same set as
+            // `!initial` but names the thing the rules actually depend on: on the first fetch there is no
+            // cache to lose, `start()` has to come up on whatever the source has, and the pre-dial guard
+            // in {@link bindConnection} is what speaks for a dead first token.
+            const expiryMs = bearerExpiryMs(bearer);
+            const held = this.currentBearer;
+            if (held !== undefined) {
+                // ALREADY DEAD, whatever else is true of it. Advancement is deliberately not part of this
+                // test: a candidate expiring at now-5s does advance a held one that died at now-60s, so an
+                // advance-only rule adopts it - overwriting the cache with material nothing can dial,
+                // skipping the recoverable warning because the fetch did not throw, and arming the next
+                // read off a non-positive delay. The creds path draws its line on expiry alone too, in
+                // {@link presentableCreds} (#1572).
+                if (expiryMs <= Date.now())
+                    throw new Error("the bearer source returned a token that has already expired - nothing adopted");
+                // THE SAME BYTES BACK AGAIN from a source whose token cannot carry the next cycle: the
+                // delay it arms is non-positive, `armBearerRefresh` floors that to 5s, and the next read
+                // returns that same token - a 5s loop against the auth service for the rest of its life,
+                // with BEARER_RETRY_MS bypassed because the fetch did not FAIL. It succeeded and returned
+                // nothing new (#1561).
+                //
+                // The test is byte identity, not expiry, and not advancement. `exp` carries one second of
+                // resolution, so a key rotation re-signing the same claims under a new key - and a healthy
+                // short-TTL source read twice inside one wall-clock second - both hand back a token whose
+                // `exp` has not moved. That is genuinely re-issued material, which an advancement test
+                // refuses and this one adopts. It is the question the creds path asks with
+                // `credsFingerprint`: did the source re-issue ANYTHING (#1572).
+                if (bearer === held && expiryMs - Date.now() - CotalEndpoint.BEARER_REFRESH_MARGIN_MS <= 0)
+                    throw new Error("the bearer source re-served the token already held and it cannot carry another cycle (the auth service has not issued a fresh one) - nothing adopted");
+            }
             this.currentBearer = bearer;
-            this.armBearerRefresh(bearerExpiryMs(bearer) - Date.now() - CotalEndpoint.BEARER_REFRESH_MARGIN_MS);
+            // A non-positive delay is NOT clamped to BEARER_RETRY_MS. A deployment whose whole token TTL
+            // sits inside the margin is legitimate - `expiry-renewal` mints 5s bearers against the 60s
+            // margin - and for it the 5s floor IS the renewal cadence. Backing that source off to 15s
+            // leaves a 5s token dead for two thirds of every cycle. What made #1561 a pointless loop was
+            // the source returning nothing new, which is refused above, not the cadence itself.
+            this.armBearerRefresh(expiryMs - Date.now() - CotalEndpoint.BEARER_REFRESH_MARGIN_MS);
         }
         catch (e) {
             if (initial)
@@ -538,6 +581,35 @@ export class CotalEndpoint extends EventEmitter {
                 : "this endpoint was constructed with an empty creds string - not dialing without auth material (an empty credential is not anonymous access)");
         return CotalEndpoint.presentableCreds(this.currentCreds, { renewable: Boolean(this.credsSource) });
     }
+    /** Refuses a fetched generation there is nothing to renew FROM, so the caller's failure posture
+     *  ({@link CREDS_RETRY_MS}) applies instead of the renewal schedule (issue #1523).
+     *
+     *  A source that hands back the SAME generation past its own renewal point has not re-signed yet:
+     *  that is a missed remint, not a candidate. Adopting it re-arms from a non-positive delay, which
+     *  {@link armCredsRefresh} floors to 1s, which fetches again, which is the same generation — a 1s
+     *  read loop against a store that is already having a bad day, for the JWT's remaining 25% of life,
+     *  with the 60s backoff that exists for exactly this bypassed because the fetch did not FAIL.
+     *
+     *  The membership feed's rw-cred renewal already refuses it this way (`membership-feed.ts`,
+     *  `adoptRwCreds`: "the rw source still holds the previous generation past its renewal point");
+     *  this is the same rule on the endpoint's `delivery.creds` seam, so the two renewal paths answer
+     *  a dead source identically.
+     *
+     *  An ALREADY-EXPIRED generation is refused whatever it is: the delay is non-positive for it too,
+     *  but it is dead rather than merely due, so it is named separately and refused even when the
+     *  source keeps returning a different one. */
+    static assertRenewableGeneration(candidate, current, delayMs) {
+        const { exp } = credsClaims(candidate);
+        if (typeof exp === "number" && exp * 1000 <= Date.now())
+            throw new Error("the creds source returned an already-expired credential (its `exp` is in the past) - nothing adopted; the renewal owner has not re-signed it, or the store is serving a stale generation");
+        // Compared by GENERATION, not by envelope. `EndpointOptions.creds` takes opaque file content and
+        // promises no canonical whitespace: `jwtFromCreds` pads with `\s*` and trims, so a store, editor or
+        // filesystem round trip that adds a newline re-serves the SAME credential in bytes `===` calls
+        // different - and the refusal would be skipped for the one case it exists to catch. The envelope
+        // also carries the nkey seed, which is why `credsFingerprint` hashes the JWT instead.
+        if (current !== undefined && credsFingerprint(candidate) === credsFingerprint(current) && delayMs <= 0)
+            throw new Error("the creds source still holds the previous generation past its renewal point (the renewal owner has not re-signed it) - nothing adopted");
+    }
     /** The disposable-preflight connect bound for the EXPLICIT reload proof (D5 class-2 adoption). A
      *  rogue or unreachable candidate must resolve well UNDER the manager's delivery-admin request
      *  bound, so this stays a few seconds and never blocks the responder. */
@@ -563,8 +635,12 @@ export class CotalEndpoint extends EventEmitter {
         const id = idFromCreds(creds);
         if (id !== this.connId)
             throw new Error(`creds source returned identity ${id}, expected ${this.connId} - renewal may not swap the connection's nkey`);
+        // Before the commit, so a refusal leaves the cache holding the last generation that was renewable
+        // from (and, on the initial fetch, fails loud rather than dialling with dead material).
+        const delay = credsRenewalDelayMs(creds);
+        CotalEndpoint.assertRenewableGeneration(creds, this.currentCreds, delay);
         this.currentCreds = creds;
-        this.armCredsRefresh(credsRenewalDelayMs(creds));
+        this.armCredsRefresh(delay);
         const { iat, exp } = credsClaims(creds);
         return { iat, exp };
     }
@@ -711,6 +787,7 @@ export class CotalEndpoint extends EventEmitter {
         // lacking a numeric `exp`, and that throw must not leave currentCreds flipped to a candidate the
         // authenticator would present on the next reconnect (a post-preflight validation failure is a no-op).
         const delay = credsRenewalDelayMs(candidate);
+        CotalEndpoint.assertRenewableGeneration(candidate, this.currentCreds, delay);
         this.currentCreds = candidate;
         this.armCredsRefresh(delay);
         return credsClaims(candidate);
@@ -1085,6 +1162,7 @@ export class CotalEndpoint extends EventEmitter {
         this.membershipFeedKv = undefined;
         this.deliveryKv = undefined;
         this.managerLeaseKv = undefined;
+        this.daemonRenewalLeaseRevision = undefined;
         // Handles that armDeliveryControl created on a previous connection: null them so a rebind
         // does not carry a dead protocol's subs forward. Best-effort unsubscribe first (the sub is
         // dead with its connection either way; unsubscribing an already-dead sub is a noop).
@@ -1234,6 +1312,7 @@ export class CotalEndpoint extends EventEmitter {
             // The manager's liveness-lease handle too: left bound to the old connection, every renew and
             // re-read after a reconnect times out, and the manager reports its lease unknown for good.
             this.managerLeaseKv = undefined;
+            this.daemonRenewalLeaseRevision = undefined;
             // This is an application-requested epoch teardown, not a transient nats.js blip. The old
             // status iterator is now stale by construction and its close is epoch-dropped, so this line is
             // the authoritative raw-liveness edge for the no-nc window until the new watcher seeds true.
@@ -3115,8 +3194,25 @@ export class CotalEndpoint extends EventEmitter {
         this.deliveryKv ??= await openDeliveryRegistry(this.nc, this.space);
         return this.deliveryKv;
     }
+    /** This endpoint instance's LEASE INCARNATION: which run of this principal a lease row was written
+     *  by. Minted per construction and never re-derived, because the credential cannot supply it, the
+     *  daemon's cred is a file on disk that every restart re-reads, so `card.id` is stable across
+     *  processes by design. See {@link DeliveryLeaseInfo.incarnation}. */
+    leaseIncarnation = randomUUID();
     encodeLease(ready) {
-        return new TextEncoder().encode(JSON.stringify({ holder: this.card.id, since: Date.now(), ready }));
+        return new TextEncoder().encode(JSON.stringify({ holder: this.card.id, incarnation: this.leaseIncarnation, since: Date.now(), ready }));
+    }
+    /** Is this shard's lease row one THIS ENDPOINT INSTANCE wrote? The question a daemon whose renew
+     *  just failed has to answer before it decides whether it still owns the shard.
+     *
+     *  Both halves are required. `holder` alone is not sufficient (a successor daemon re-reading the
+     *  same creds file presents the same principal, so its row would read as ours), and `incarnation`
+     *  alone is not sufficient either, it is a bare uuid with no claim to the principal, so a row
+     *  bearing ours but a foreign holder is not something we should ever adopt. A row with NO
+     *  incarnation predates the field and cannot be proven ours, which is the safe reading: it leads
+     *  to the takeover path rather than to serving on someone else's claim. */
+    ownsDeliveryLease(info) {
+        return info.holder === this.card.id && info.incarnation !== undefined && info.incarnation === this.leaseIncarnation;
     }
     /** Acquire the single-flight delivery lease for a shard via an ATOMIC CAS create, marked NOT-ready.
      *  THROWS if a live lease exists — a loud refusal-to-bind (the daemon exits), never a retry, so two
@@ -3132,6 +3228,18 @@ export class CotalEndpoint extends EventEmitter {
     async markDeliveryLeaseReady(shardIndex, revision) {
         return (await this.deliveryRegistry()).update(leaseKey(shardIndex), this.encodeLease(true), revision);
     }
+    /** Flip the held lease back to NOT-ready, the counterpart to {@link markDeliveryLeaseReady}, for a
+     *  holder that has UNBOUND its loops and control responder but has not given up the shard.
+     *
+     *  `ready` is a claim about the RESPONDER, not about the row's existence: `ensureDelivery` waits on
+     *  it and the channel-health surface reports it. A daemon that goes quiet to re-check its ownership
+     *  still holds the key, so without this the space would be told a responder is up while nothing is
+     *  bound, a readiness lie of exactly the kind #1318 is about, just pointed the other way. Keeping
+     *  the row (rather than deleting it) is deliberate: the shard is still claimed, so no third daemon
+     *  should be invited in; what is being withdrawn is only the claim to be answering. */
+    async markDeliveryLeaseNotReady(shardIndex, revision) {
+        return (await this.deliveryRegistry()).update(leaseKey(shardIndex), this.encodeLease(false), revision);
+    }
     /** Renew the held lease (CAS `kv.update` against `revision`, keeping `ready:true`) to refresh it before
      *  the bucket TTL expires it. Returns the new revision. Throws if the revision moved (lost the lease —
      *  the daemon should exit). */
@@ -3139,25 +3247,54 @@ export class CotalEndpoint extends EventEmitter {
         return (await this.deliveryRegistry()).update(leaseKey(shardIndex), this.encodeLease(true), revision);
     }
     /** Release the held lease on clean shutdown so a replacement daemon re-acquires immediately (best
-     *  effort — a crash just lets the bucket TTL expire it). */
-    async releaseDeliveryLease(shardIndex) {
+     *  effort, a crash just lets the bucket TTL expire it).
+     *
+     *  THE REVISION IS WHAT MAKES THIS A RELEASE RATHER THAN A DELETE. An unconditional delete removes
+     *  whatever row is there, and by shutdown time the row is not necessarily still ours: the exit
+     *  paths that matter most are precisely the ones where another daemon has taken the shard, so the
+     *  departing process would delete the REPLACEMENT's lease on its way out and leave the shard with
+     *  no holder at all. Passing the revision this endpoint last owned makes the delete a compare-and-
+     *  swap (`previousSeq` becomes JetStream's `ExpectedLastSubjectSequence`), so a row that has moved
+     *  on is left alone.
+     *
+     *  THE ARGUMENT IS REQUIRED, AND EXPLICITLY NULLABLE RATHER THAN OPTIONAL. `undefined` means "this
+     *  process no longer holds a revision it can argue for", which is the takeover paths' honest
+     *  answer and correctly releases nothing, the bucket TTL is the crash-safe authority and expires
+     *  a genuinely stale row. But if that were the DEFAULT, every existing `releaseDeliveryLease(0)`
+     *  call site would keep compiling and silently stop releasing: the same omission hole
+     *  `standaloneConnectOpts` closed by deleting its `= {}`. Measured, not theorised, this landed as
+     *  a red in `smoke:delivery-lease`, where a caller that genuinely held the lease released nothing
+     *  and the next acquire was refused. A caller must now say which it means. */
+    async releaseDeliveryLease(shardIndex, revision) {
+        if (revision === undefined)
+            return;
         try {
-            await (await this.deliveryRegistry()).delete(leaseKey(shardIndex));
+            await (await this.deliveryRegistry()).delete(leaseKey(shardIndex), { previousSeq: revision });
         }
         catch {
             // Intentionally best-effort for EVERY failure: the lease TTL is the crash-safe release authority,
-            // and clean shutdown must continue even when the broker is already gone or draining.
+            // and clean shutdown must continue even when the broker is already gone or draining. A refused
+            // CAS lands here too, which is the correct outcome, someone else owns the row.
         }
     }
     /** Read a shard's delivery lease (the daemon-availability signal), or `undefined` if none is live.
      *  READ-ONLY surface — drives Component 6's `cotal_channels` delivery-health field (an agent reads it
      *  under its own cred, which holds lease-bucket read but no write). */
     async readDeliveryLease(shardIndex) {
+        return (await this.readDeliveryLeaseEntry(shardIndex))?.info;
+    }
+    /** The lease row AND the KV revision it is at. The revision is the CAS token every renew and the
+     *  CAS release are argued against, so a caller re-establishing ownership after a failed renew
+     *  needs the BROKER's sequence, not the one it last cached: a renew can fail with its write
+     *  already applied (a lost reply, a reconnect mid-request), which leaves the cached revision one
+     *  behind forever and every subsequent CAS refused over a sequence this process itself moved ,
+     *  read as somebody else's takeover, which is the #1318 misreading in a second costume. */
+    async readDeliveryLeaseEntry(shardIndex) {
         const e = await (await this.deliveryRegistry()).get(leaseKey(shardIndex));
         if (!e || e.operation === "DEL" || e.operation === "PURGE")
             return undefined;
         try {
-            return e.json();
+            return { info: e.json(), revision: e.revision };
         }
         catch {
             return undefined;
@@ -3198,6 +3335,53 @@ export class CotalEndpoint extends EventEmitter {
         }
         return this.managerLeaseKv;
     }
+    /** Take or keep the per-SPACE daemon-credential renewal lease (#1634), returning whether THIS
+     *  instance now holds it. One atomic CAS `create` per pass: it succeeds for whoever arrives first
+     *  and throws for everyone else, so exactly one manager remints even when several share the
+     *  daemon's store. The holder re-`update`s its own key by revision, which both keeps it and proves
+     *  it never lost it. Losing the CAS is an ordinary outcome (a peer holds it) and returns false; it
+     *  never fails a start. A crashed holder's key TTL-expires with the bucket, so the next pass hands
+     *  the lease to a survivor with no operator step. */
+    async holdDaemonRenewalLease(instanceId) {
+        const kv = await this.managerLeaseRegistry();
+        const held = this.daemonRenewalLeaseRevision;
+        if (held !== undefined) {
+            try {
+                this.daemonRenewalLeaseRevision = await kv.update(MANAGER_RENEWAL_LEASE_KEY, this.encodeDaemonRenewalLease(instanceId), held);
+                return true;
+            }
+            catch {
+                // The revision moved (our key TTL-expired and a peer took it). Re-contend below rather than
+                // keep reminting on a lease we no longer hold.
+                this.daemonRenewalLeaseRevision = undefined;
+            }
+        }
+        try {
+            this.daemonRenewalLeaseRevision = await kv.create(MANAGER_RENEWAL_LEASE_KEY, this.encodeDaemonRenewalLease(instanceId));
+            return true;
+        }
+        catch {
+            return false; // another manager holds it: exactly one owner is the point
+        }
+    }
+    /** Release the renewal lease on a clean stop so a peer takes over at once rather than at the TTL.
+     *  CAS-guarded, so a lease we already lost is never deleted out from under its new holder. */
+    async releaseDaemonRenewalLease() {
+        const held = this.daemonRenewalLeaseRevision;
+        this.daemonRenewalLeaseRevision = undefined;
+        if (held === undefined)
+            return;
+        try {
+            await (await this.managerLeaseRegistry()).delete(MANAGER_RENEWAL_LEASE_KEY, { previousSeq: held });
+        }
+        catch {
+            // Best-effort, like releaseManagerLease: a moved revision means it is not ours, and a broker
+            // failure is recovered by the bucket TTL. Shutdown must not claim deletion.
+        }
+    }
+    encodeDaemonRenewalLease(instanceId) {
+        return new TextEncoder().encode(JSON.stringify({ instanceId, since: Date.now() }));
+    }
     encodeManagerLease(info) {
         return new TextEncoder().encode(JSON.stringify(info));
     }
@@ -3529,7 +3713,28 @@ export class CotalEndpoint extends EventEmitter {
      *  id is the authenticated subject sender ({@link serveControl} fail-closes on a mismatch). Validation
      *  is against the durable ACL registry — the SAME KV the reader re-auths against (single source of
      *  truth, no in-memory ledger to drift). */
+    /** Whether an ALREADY-DISPATCHED unit of Plane-3 work may still take effect.
+     *
+     *  Unsubscribing stops NEW work; it cannot recall work already in flight. A handler that entered
+     *  before {@link quiescePlane3} and awaited broker I/O inside it resumes AFTER the freeze, and a
+     *  reviewer traced the ordering that makes that a split rather than a latency blip: the loser
+     *  accepts a unit and awaits, the loser is descheduled, the successor acquires the shard and flips
+     *  its lease READY, then the loser resumes and answers or acks for a shard it no longer holds.
+     *  Subscription counts and parked-pull readings cannot see it, because the effect is the reply and
+     *  the ack rather than the binding.
+     *
+     *  So each work path re-asks HERE, at the point of effect, after its awaits and before it acts.
+     *  The lease keeps one ROW, not one SERVER; this is the half that keeps one server. An unacked
+     *  message is not lost by refusing: a stopped consumer redelivers it to whoever holds the shard
+     *  next, which is the same property quiescing already relies on. */
+    plane3MayAct() {
+        return !this.plane3Quiesced;
+    }
     async handleDeliveryControl(req) {
+        // FENCE: entered before a quiesce, resuming after it. Answering now would put a second server on
+        // this shard's control rail while the winner is already READY.
+        if (!this.plane3MayAct())
+            return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
         const caller = req.from.id;
         const args = req.args ?? {};
         if (req.op === "durableJoin")
@@ -3548,11 +3753,18 @@ export class CotalEndpoint extends EventEmitter {
             catch (e) {
                 return { ok: false, error: e.message };
             }
-            return { ok: true, data: { memberships: await this.ownerMemberships(caller, uid) } };
+            const memberships = await this.ownerMemberships(caller, uid);
+            // FENCE AFTER THE AWAIT, which is the one that matters. The entry check above cannot catch the
+            // case this exists for: a request admitted WHILE SERVING, then descheduled inside its broker
+            // read, resuming after a successor has taken the shard and reached READY. Re-ask at the point
+            // of effect, so the answer is never served by a daemon that has stood down.
+            if (!this.plane3MayAct())
+                return { ok: false, error: "delivery: this daemon stopped serving this shard while the request was in flight; retry" };
+            return { ok: true, data: { memberships } };
         }
         return { ok: false, error: `op "${req.op}" not supported on the delivery control service` };
     }
-    /** Validate the channel ARG shape only — non-blank, valid, concrete (NO ACL check, that is op-specific).
+    /** Validate the channel ARG shape only: non-blank, valid, concrete (NO ACL check, that is op-specific).
      *  Returns the channel on success or a ControlReply error to short-circuit. */
     checkDurableChannelArg(args, op) {
         const channel = typeof args.channel === "string" ? args.channel.trim() : "";
@@ -3735,6 +3947,85 @@ export class CotalEndpoint extends EventEmitter {
         const max = this.nc?.info?.max_payload ?? 1_048_576;
         return Math.max(1, Math.floor(max * 0.9));
     }
+    /** Stop serving Plane-3 WITHOUT tearing down the connection, so a daemon that has just learned its
+     *  lease may no longer be its own can stop acting on the shard while it finds out for certain.
+     *
+     *  A COMPARE-AND-SWAP KEEPS ONE LEASE ROW; IT DOES NOT KEEP ONE SERVER. That distinction is the
+     *  reason this exists, and it was a review finding. When a renew fails, the daemon re-reads the
+     *  key and may then re-acquire it, and across that read-then-create it was still consuming the
+     *  fan-out durable, still running the inbox reader, and still answering ctl.delivery. If a
+     *  replacement acquired the shard in that window, both processes served the same durables until
+     *  the loser's create was refused and its teardown finished. The old code did not have this
+     *  window, because it began shutting down on the first renew failure; treating that failure as a
+     *  question instead of a verdict is right, but asking the question while still serving is not.
+     *
+     *  So the daemon goes quiet FIRST and re-arms only once it has proof: `held` on a re-read, or a
+     *  won atomic create. `unknown` stays quiet, the whole point is that not being able to ask is not
+     *  permission to keep acting. Quiescing costs delivery latency for a few seconds; the alternative
+     *  costs a SPLIT durable, which is a correctness failure rather than an availability one.
+     *
+     *  Deliberately not `stop()`: the connection, the lease KV handles and the control rails must stay
+     *  up, because the daemon still has to ask the broker who owns the shard. */
+    async quiescePlane3() {
+        this.plane3Quiesced = true;
+        if (this.deliveryServeSub) {
+            try {
+                this.deliveryServeSub.unsubscribe();
+            }
+            catch { /* already dead */ }
+            this.deliveryServeSub = undefined;
+        }
+        if (this.deliveryAdminServeSub) {
+            try {
+                this.deliveryAdminServeSub.unsubscribe();
+            }
+            catch { /* already dead */ }
+            this.deliveryAdminServeSub = undefined;
+        }
+        // Stopping the consumers ends the `for await` loops that drive fan-out and the reader. In-flight
+        // messages are NOT acked by a stopped consumer, so they redeliver to whoever holds the shard
+        // next: quiescing loses no message, it only stops this process from claiming them.
+        for (const msgs of this.streamMsgs.splice(0)) {
+            try {
+                msgs.stop();
+            }
+            catch { /* already draining */ }
+        }
+    }
+    /** Whether Plane-3 is currently quiesced: unbound, and refusing to re-bind until ownership is proven.
+     *  Read-only. The flag is the daemon's own answer to "am I serving this shard", so a cell that asserts
+     *  a recovery worked can check the endpoint's state rather than inferring it from a log line. */
+    plane3IsQuiesced() {
+        return this.plane3Quiesced;
+    }
+    /** Resume serving Plane-3 after {@link quiescePlane3}, once ownership has been re-established.
+     *  Idempotent, and a no-op when the daemon was never quiesced.
+     *
+     *  The flag is cleared ONLY after every binding is up, and restored if any of them throws. Clearing
+     *  it first looks equivalent and is not: `armPlane3` binds in four stages (`manager`, the two control
+     *  responders, the fan-out consumer, the reader), so a failure at any one leaves the endpoint recorded
+     *  as un-quiesced while some of those are missing. From there every later `rearmPlane3` returns at the
+     *  `!plane3Quiesced` guard WITHOUT attempting to bind, and the caller goes on to flip the lease READY.
+     *  That is a readiness lie surviving a transient broker error, the daemon claims a responder it does
+     *  not have, which is the #1318 outage wearing the readiness flag instead of the exit path. */
+    async rearmPlane3() {
+        if (!this.plane3Quiesced)
+            return;
+        // `armPlane3` refuses while quiesced (a reconnect must not resume mid-question), so the flag is
+        // lowered for the attempt and RAISED AGAIN on failure. A partial bind leaves the endpoint quiesced,
+        // which is the honest state: not serving, and a later retry will attempt the whole sequence again.
+        this.plane3Quiesced = false;
+        try {
+            await this.armPlane3();
+        }
+        catch (e) {
+            this.plane3Quiesced = true;
+            // Drop whatever DID bind. Leaving half-bound responders up would answer control requests for a
+            // shard this process has just declared itself not to be serving.
+            this.quiescePlane3();
+            throw e;
+        }
+    }
     /** (Re)bind the Plane-3 fan-out writer + trusted reader. Idempotent — the durables resume from their
      *  cursor. Called by {@link startPlane3} once AND by {@link connectAndBind} on every (re)connect, so
      *  the delivery daemon's reconnect RE-ARMS the backstop + the ctl.delivery responder. Without this, a broker blip would silently kill
@@ -3743,6 +4034,11 @@ export class CotalEndpoint extends EventEmitter {
     async armPlane3() {
         if (!this.plane3 || !this.js)
             return;
+        // A quiesced endpoint must not be re-armed by a RECONNECT: the reconnect path calls this too,
+        // and silently resuming there would restore exactly the double-serving this guards against
+        // while the ownership question is still open.
+        if (this.plane3Quiesced)
+            return;
         await this.manager(); // the manager runs consume:false, so this.jsm is lazy — ensure it
         this.armDeliveryControl();
         await this.runFanout();
@@ -3783,6 +4079,9 @@ export class CotalEndpoint extends EventEmitter {
      *  feed's rw connection, and reply with proof (identities + the adopted JWT windows) — or a
      *  structured failure (e.g. the file was never re-signed), never a silent partial. */
     async handleDeliveryAdmin(req) {
+        // FENCE: same reason as the runtime rail above.
+        if (!this.plane3MayAct())
+            return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
         if (req.op === "reloadCreds") {
             // The renewal owner's EXPECTED-generation tokens (SHA-256 of each JWT it re-signed), per
             // component. A missing entry means "no expectation" (the passive backstop still adopts).
@@ -3911,6 +4210,13 @@ export class CotalEndpoint extends EventEmitter {
      *  members within interval; `live` channel → `@mention` targets authorized to read it (ACL only).
      *  Members KV is scanned FRESH per message (no cache — red-team BLOCKER-1 catch-up correctness). */
     async fanOutMessage(m) {
+        // FENCE, and NOT followed by an ack: this unit was dispatched before the quiesce and is resuming
+        // after it, so writing the fan-out now would put entries in member inboxes on behalf of a shard
+        // this daemon no longer serves. Returning WITHOUT acking is the whole point - a stopped consumer
+        // redelivers to whoever holds the shard next, so refusing here loses no message, while acking it
+        // would consume the successor's work.
+        if (!this.plane3MayAct())
+            return;
         const parsed = parseSubject(m.subject);
         if (!parsed || parsed.kind !== "chat") {
             m.ack();
@@ -3931,6 +4237,12 @@ export class CotalEndpoint extends EventEmitter {
         } // authenticity (owner must be a real principal, not an old-shape alias)
         const seq = m.seq;
         const normalizedMsg = authenticatedChannelMessage(msg, channel);
+        // SECOND FENCE, AT THE COMMIT POINT. The entry check above cannot cover this unit's own awaits:
+        // the class read and the member scan below both hit the broker, and a daemon descheduled inside
+        // them can resume after a successor owns the shard. Re-ask immediately before the first WRITE,
+        // and again return without acking so the entry redelivers rather than being consumed here.
+        if (!this.plane3MayAct())
+            return;
         if ((await this.deliveryClassFresh(channel)) === "durable") {
             for (const rec of await listMembers(await this.membersRegistry(), { channel })) {
                 if (rec.owner === msg.from.id)
@@ -3941,6 +4253,12 @@ export class CotalEndpoint extends EventEmitter {
                 // retired lifecycle's inbox, never the alias's new occupant (SPEC 13.1 cross-plane scoping).
                 // Store the AUTHENTICATED-channel copy (main's normalization): the durable frame validates
                 // msg.channel === frame.channel, and payload to/toService are stripped.
+                // Per-copy, because the scan above and every publish below is a broker round trip: a fan-out
+                // to fifty members can straddle a quiesce in its middle. Stopping part-way is safe where
+                // stopping late is not - the entry is never acked here, so the successor redelivers it and
+                // re-sends the whole set; publishDinbox is idempotent per (owner, lifecycle, msgID).
+                if (!this.plane3MayAct())
+                    return;
                 await this.publishDinbox(rec.owner, rec.lifecycleUid, { msg: normalizedMsg, channel, seq, reason: "durable-channel", generation: rec.generation });
             }
         }
@@ -3962,9 +4280,16 @@ export class CotalEndpoint extends EventEmitter {
                 }
                 if (!row || !channelInAllow(row.allowSubscribe, channel))
                     continue; // @mention can't bypass the read ACL
+                if (!this.plane3MayAct())
+                    return; // same rule on the live-mention path
                 await this.publishDinbox(owner, row.lifecycleUid, { msg: normalizedMsg, channel, seq, reason: "live-mention", generation: 0 });
             }
         }
+        // FINAL FENCE, BEFORE THE ACK ITSELF, which is the real commit: acking tells the stream this
+        // shard's work is done. A daemon that stopped serving mid-fan-out must not make that claim, or
+        // the successor never sees the entry and the message is lost outright rather than duplicated.
+        if (!this.plane3MayAct())
+            return;
         m.ack();
     }
     /** Trusted-reader loop: bind the single privileged `reader` durable over `dinbox.>` and re-authorize
@@ -3998,6 +4323,10 @@ export class CotalEndpoint extends EventEmitter {
      *  revoked/narrowed ACL or out-of-interval seq; on transfer success, ack the mixed entry (durability
      *  has moved to DLV — an §8 equivalent per-member at-least-once mechanism). The agent acks DLV. */
     async readerHandle(m) {
+        // FENCE, same rule as the fan-out writer: no effect and no ack once this daemon has stopped
+        // serving the shard. The entry redelivers to the holder rather than being consumed here.
+        if (!this.plane3MayAct())
+            return;
         const pr = parseDinboxPrincipal(m.subject);
         if (!pr) {
             m.ack();
@@ -4047,6 +4376,12 @@ export class CotalEndpoint extends EventEmitter {
                 return;
             }
         }
+        // COMMIT-POINT FENCE. Everything above this line is broker I/O - the ACL read and the member
+        // re-read - so the entry check at the top of the handler is stale by now if this unit was
+        // descheduled inside either. The transfer below is the effect that must not happen on behalf of
+        // a shard we no longer serve; returning without acking leaves it for the holder.
+        if (!this.plane3MayAct())
+            return;
         try {
             // DLV has no original chat subject, so preserve the channel the trusted fan-out reader derived
             // from CHAT. Never let the publisher-controlled payload label choose connector attention.
@@ -4077,6 +4412,12 @@ export class CotalEndpoint extends EventEmitter {
             m.nak(2000);
             return;
         }
+        // The ack is the last commit: the transfer above already happened, so the only question left is
+        // whether THIS daemon may retire the source entry. If it stopped serving mid-transfer, leave the
+        // entry pending - the successor re-transfers and stream-wide dedupe on the msgID collapses the
+        // duplicate, which is the recoverable direction. Acking here is the unrecoverable one.
+        if (!this.plane3MayAct())
+            return;
         m.ack();
     }
     /** Agent-side: bind + pump our pre-created Plane-3 DELIVER durable (`dlv_<id>`). Every message here is
@@ -5031,9 +5372,9 @@ export class CotalEndpoint extends EventEmitter {
      *  every window, one consumer create and one warning per TTL for as long as the mesh was empty.
      *
      *  A REGISTERING observer (the manager) is itself one of the keys that should be there. An
-     *  empty bucket under it means the bucket was wiped since its last heartbeat (the netcup
+     *  empty bucket under it means the bucket was wiped since its last heartbeat (the stream
      *  recreation), and the same wipe took every peer's record: their absence says the bucket is
-     *  new, not that they left. rev-1421-gpt reproduced the previous behaviour at default timing:
+     *  new, not that they left. a reviewer reproduced the previous behaviour at default timing:
      *  the rebind landed ~0.9s after the recreation, the observer marked every peer AND ITSELF
      *  offline, and held the view current for up to one heartbeat, a false verdict `cotal ps`
      *  would print as `mesh offline`. So a registering observer re-publishes its own record NOW,
@@ -5186,7 +5527,7 @@ export class CotalEndpoint extends EventEmitter {
         if (this.lastPresenceWatchAt !== 0 && now - this.lastPresenceWatchAt > this.ttlMs) {
             this.emitPresenceViewIfChanged();
             // Staying stale is the right verdict for a held link (#1045), and the wrong END STATE when
-            // the transport is up and the watch's own consumer is what died. Measured on netcup
+            // the transport is up and the watch's own consumer is what died. Measured on a live deployment
             // 2026-09-09: the presence stream was deleted and recreated, its sequence restarted, and
             // every observer's ORDERED consumer re-created itself at the OLD start sequence (nats.js
             // 3.4.0 resets from its cursor). The broker kept sending idle heartbeats, so the client
@@ -5508,8 +5849,14 @@ export function wsServers(servers) {
  *  probe dials; a ws(s) broker is by definition published through an HTTPS edge (CDN tunnel,
  *  reverse proxy), where TLS + upgrade + INFO + the auth round-trip routinely exceeds 1s cold —
  *  measured ~60% spurious "not reachable" against a Cloudflare-fronted broker. Callers passing an
- *  explicit `timeoutMs` are untouched. */
-function defaultProbeTimeoutMs(servers) {
+ *  explicit `timeoutMs` are untouched.
+ *
+ *  EXPORTED because a caller that judges a probe by WHEN it answered has to compare against the
+ *  deadline this function actually handed the probe. The delivery watchdog does exactly that, and
+ *  hardcoding 1000 there silently misread every ws broker: honest refusals arrive at 2-5s, past a
+ *  budget that was never theirs, and would be classified as this process's starvation rather than
+ *  the server's refusal. The budget and the judgment must come from one place. */
+export function defaultProbeTimeoutMs(servers) {
     return wsServers(servers) ? 5000 : 1000;
 }
 /** Pick the dial function by SCHEME: `ws://`/`wss://` servers go through nats-core's