@cotal-ai/core 0.48.2 → 0.49.0

package/dist/endpoint.js

@@ -5,14 +5,16 @@ import { connect, credsAuthenticator, headers, tokenAuthenticator, nanos, Author
 import { wsconnect } from "@nats-io/nats-core";
 import { credsClaims, credsFingerprint, credsRenewalDelayMs, idFromCreds } from "./identity.js";
 import { inspectCredHealth } from "./provision.js";
+import { parseSecretStoreIdentity, } from "./secret-store.js";
 import { resolveService, invokeCommand, submitAndFollowGoal } from "./endpoint-invoke.js";
 import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, EP_BIND_REFUSED } from "./endpoint-envelope.js";
 import { isRepeatSafeCommand } from "./endpoint-grants.js";
-import { assertIdToken } from "./endpoint-subjects.js";
+import { assertIdToken, assertGeneration } from "./endpoint-subjects.js";
+import { readAcceptedRow } from "./issued-authority.js";
 import { liveKvEntries } from "./kv-scan.js";
 import { ARTIFACT_PART_KIND, isArtifactPart } from "./artifact.js";
 import { assertValidName } from "./resolve.js";
-import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MAX_MSGS_PER_SUBJECT, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS } from "./streams.js";
+import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MAX_MSGS_PER_SUBJECT, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, TTL_RECONCILE_CANARY_KEY } from "./streams.js";
 import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, JetStreamApiError, } from "@nats-io/jetstream";
 import {} from "@nats-io/jetstream";
 import { Kvm } from "@nats-io/kv";
@@ -112,6 +114,12 @@ export class CotalEndpoint extends EventEmitter {
     /** The freshest bearer — what every (re)connect attempt presents. */
     currentBearer;
     bearerTimer;
+    /** Arms against the credential authenticated on the current wire. nats-core 3.4.0 discards the
+     *  promise returned by its async transport-close continuation; if that continuation enters the
+     *  reconnect dial loop with an expired JWT, its terminal auth error becomes an unhandled rejection.
+     *  Cotal knows the JWT expiry, so it disables the library reconnect before the broker closes the
+     *  transport. The endpoint's observed `closed()` supervisor remains responsible for rebuilding. */
+    authExpiryReconnectTimer;
     sentinelCreds;
     tls;
     heartbeatMs;
@@ -202,6 +210,10 @@ export class CotalEndpoint extends EventEmitter {
     firstConnect = true;
     heartbeatTimer;
     sweepTimer;
+    /** #1356: when the presence bucket started refusing writes; undefined once one succeeds. */
+    presenceWriteFailingSince;
+    /** #1356: the broker's last refusal message, kept alongside the start time for diagnosis. */
+    lastPresenceWriteError;
     roster = new Map();
     /** Resolves when the current presence watch has consumed its complete initial KV snapshot. */
     presenceSnapshot = Promise.resolve();
@@ -216,6 +228,18 @@ export class CotalEndpoint extends EventEmitter {
     lastPresenceWatchAt = 0;
     /** Last emitted presence-view state. Suppresses duplicate `presence-view` events. */
     presenceViewState = "unpopulated";
+    /** A presence-watch rebind in flight (see {@link rebindStalePresenceWatch}); one at a time. */
+    presenceRebind;
+    /** Bumped by every connection-scoped teardown and by {@link stop}. A presence bind that was
+     *  awaiting the broker when the epoch moved belongs to a retired epoch: it releases the
+     *  iterator it got and installs nothing (see {@link startPresenceWatch}). */
+    presenceEpoch = 0;
+    /** The current watch was bound onto a bucket with NO keys (see {@link markPresenceBucketEmpty}).
+     *  Such a watch cannot deliver until someone writes, so its silence is not staleness. */
+    presenceWatchEmpty = false;
+    /** Wall-clock of the last rebind attempt, so a bucket that is silent because it is EMPTY (or a
+     *  broker that keeps refusing the consumer create) is retried once per TTL, not per sweep tick. */
+    presenceRebindAt = 0;
     status = "idle";
     activity;
     /** Mirror of the connector's authoritative attention state, published in presence (advisory). The
@@ -256,6 +280,9 @@ export class CotalEndpoint extends EventEmitter {
     actorIsEphemeral;
     /** This incarnation's lifecycle UID (opts.lifecycleUid) — see {@link EndpointOptions.lifecycleUid}. */
     ownLifecycleUid;
+    acceptedToken;
+    /** The issuer-bound generation, learned once per connection from the accepted row. */
+    issuedGeneration;
     /** Per-endpoint-name {@link resolveService} cache for {@link invokeService} — dropped on a
      *  `failed-precondition` currency refusal (the described incarnation was superseded). */
     resolvedServices = new Map();
@@ -376,6 +403,11 @@ export class CotalEndpoint extends EventEmitter {
                 : this.authed
                     ? undefined
                     : mintLifecycleUid();
+        if (opts.acceptedToken !== undefined) {
+            if (!opts.creds)
+                throw new Error("EndpointOptions.acceptedToken names a static issuance and needs creds beside it (SPEC 13.15)");
+            this.acceptedToken = assertGeneration(opts.acceptedToken, "acceptedToken");
+        }
         // `card.id` is the principal DOT-FORM `<owner>.<actor>` — the wire identity every `from.id` carries;
         // principalKey validates both tokens.
         const principal = principalKey(this.owner, this.actor);
@@ -464,6 +496,48 @@ export class CotalEndpoint extends EventEmitter {
     /** How soon a FAILED creds refresh retries. Successful refreshes schedule by lifetime fraction
      *  (75% of iat→exp), not a fixed margin — standing creds span hours to days, bearers minutes. */
     static CREDS_RETRY_MS = 60_000;
+    /** THE ONE PLACE a credential is cleared for presentation to a broker.
+     *
+     *  The property is unconditional — this endpoint never presents a credential it has already
+     *  decoded as expired — so it is a property of the SUPPLY, not of any one dial site. It used to
+     *  live inside {@link bindConnection}, which only `start()` and `doRebuild` reach; the
+     *  authenticator nats.js re-evaluates on ITS OWN reconnects read the cache directly and so
+     *  presented whatever was last fetched, expired included. Two such reconnects exist and neither
+     *  passes through bindConnection: the one the broker forces at JWT `exp`, and an ALREADY RUNNING
+     *  dial loop from an earlier drop that crosses `exp` while it retries (the pre-expiry
+     *  reconnect fence flips a policy flag nats-core only reads when it observes a NEW drop, so it
+     *  cannot stop a loop already in flight).
+     *
+     *  Putting the refusal here instead means a future caller cannot miss it: the only way to reach a
+     *  dial is through {@link credsForWire}, and the one presentation that does not read the cache
+     *  (the adoption preflight, which presents a fresh CANDIDATE) calls this same function on it.
+     *
+     *  An unbounded credential (no numeric `exp`) is presentable: bounded lifetimes are the renewal
+     *  seam's concern, and a cred with no expiry has none to be past. */
+    static presentableCreds(creds, opts) {
+        const { exp } = credsClaims(creds); // throws on a structurally-unusable file (fail-loud)
+        if (typeof exp === "number" && exp * 1000 <= Date.now())
+            throw new Error(opts.renewable
+                ? "this endpoint's creds have expired and renewal is failing - not presenting the expired credential to the broker; retrying with backoff"
+                : "this endpoint's creds have expired and it holds no creds source to renew them - replace the credential and rebuild the endpoint (pass a creds FUNCTION for standing renewal)");
+        return creds;
+    }
+    /** The cached credential, checked. Handed to nats.js as the authenticator's source on EVERY auth
+     *  mode (renewed or static), so each (re)connect attempt — ours or the library's — re-reads a
+     *  CHECKED value. A refusal throws out of the authenticator, which nats-core turns into a closed
+     *  connection rather than a CONNECT carrying dead material; the endpoint's own supervisor then
+     *  rebuilds on capped backoff, and {@link bindConnection} re-fetches from the source on each of
+     *  those attempts, so a renewal that starts working recovers the endpoint without presenting
+     *  anything expired in the meantime. Deliberately side-effect free: kicking the renewal timer
+     *  from here would retry the source once per dial attempt, which is the flat load on a dead
+     *  broker that {@link RETRY_BACKOFF_CAP_MS} exists to prevent. */
+    credsForWire() {
+        if (!this.currentCreds)
+            throw new Error(this.credsSource
+                ? "this endpoint has no credential to present yet (the creds source has not returned one) - not dialing without auth material"
+                : "this endpoint was constructed with an empty creds string - not dialing without auth material (an empty credential is not anonymous access)");
+        return CotalEndpoint.presentableCreds(this.currentCreds, { renewable: Boolean(this.credsSource) });
+    }
     /** The disposable-preflight connect bound for the EXPLICIT reload proof (D5 class-2 adoption). A
      *  rogue or unreachable candidate must resolve well UNDER the manager's delivery-admin request
      *  bound, so this stays a few seconds and never blocks the responder. */
@@ -497,13 +571,62 @@ export class CotalEndpoint extends EventEmitter {
     /** Swap the live connection onto the freshest cached cred with a controlled `nc.reconnect()`
      *  (nats.js re-evaluates the creds getter per attempt). Swapping now, instead of waiting for the
      *  broker to close the connection at `exp`, means the wire never carries a near-dead JWT and the
-     *  operator never sees a spurious "authentication expired" — the broker's expiry-close remains the
-     *  BACKSTOP if a swap is missed, not the mechanism. Already-closed/draining rejections are the
-     *  supervise loop's to own (its rebuild re-fetches); an already-disconnected client is a no-op
-     *  (its own reconnect loop presents the fresh cred). */
+     *  operator never sees a spurious "authentication expired". After the reconnect succeeds, re-arm
+     *  the expiry fence for the credential now authenticated on the wire. */
     async swapConnectionOntoFreshCreds() {
-        if (this.nc && !this.stopped)
-            await this.nc.reconnect().catch(() => { });
+        const nc = this.nc;
+        if (nc && !this.stopped) {
+            await nc.reconnect().catch(() => { });
+            if (this.nc === nc && !this.stopped)
+                this.armAuthExpiryReconnectFence(nc);
+        }
+    }
+    /** nats-core keeps its reconnect switch on the protocol handler. This pinned internal shape is the
+     *  same last-resort surface used by {@link closeFailedBind}; there is no public API for changing the
+     *  reconnect policy of an existing connection. Disabling it does not disable Cotal self-heal: the
+     *  `nc.closed()` supervisor below rebuilds the endpoint with freshly checked auth material. */
+    disableLibraryReconnect(nc) {
+        const protocol = nc.protocol;
+        if (protocol?.options)
+            protocol.options.reconnect = false;
+    }
+    /** Close a connection whose library reconnect has already been disabled. `drain()` flushes with a
+     *  PING and waits for the matching PONG; nats-core only rejects that waiter inside reconnect
+     *  `prepare()`, so a half-open socket with reconnect=false leaves drain pending until the 2-minute
+     *  ping interval times out. `close()` tears the transport down without that round-trip, which is
+     *  the same public path {@link closeFailedBind} already uses when there is no graceful delivery
+     *  contract left. */
+    async closeWithoutLibraryReconnect(nc) {
+        if (!nc)
+            return;
+        this.disableLibraryReconnect(nc);
+        try {
+            await nc.close();
+        }
+        catch {
+            /* already closing */
+        }
+    }
+    /** Disable nats-core reconnect shortly before the JWT authenticated on this wire expires. The small
+     *  lead makes the policy change precede the broker's expiry close even when both timers wake in the
+     *  same event-loop turn. A credential adoption does not move this fence until the resident reconnect
+     *  is requested, so the old wire remains protected during the prove-then-adopt window. */
+    armAuthExpiryReconnectFence(nc) {
+        clearTimeout(this.authExpiryReconnectTimer);
+        this.authExpiryReconnectTimer = undefined;
+        const credsExp = this.currentCreds && credsClaims(this.currentCreds).exp;
+        const expiryMs = this.userMode && this.currentBearer
+            ? bearerExpiryMs(this.currentBearer)
+            : typeof credsExp === "number"
+                ? credsExp * 1000
+                : undefined;
+        if (expiryMs === undefined || this.stopped)
+            return;
+        this.authExpiryReconnectTimer = setTimeout(() => {
+            if (this.nc === nc)
+                this.disableLibraryReconnect(nc);
+        }, Math.max(0, expiryMs - Date.now() - 500));
+        this.authExpiryReconnectTimer.unref?.();
     }
     /** The connectAndBind PRE-CONNECT fetch: pull the freshest source cred and pin it into
      *  {@link currentCreds} so the connect() that immediately follows presents it — that connect IS the
@@ -573,7 +696,13 @@ export class CotalEndpoint extends EventEmitter {
             throw new Error("reloadCreds: re-read credential generation did not match the expected re-signed generation (a different store, or a torn/stale read); nothing adopted");
         // PREFLIGHT = the proof. A disposable connection presenting exactly the candidate BEFORE the live
         // cache is touched; a refused cred throws here, leaving the resident connection untouched.
-        const probe = await probeConnect(this.servers, { creds: candidate, tls: this.tls, timeoutMs: Math.max(500, Math.min(CotalEndpoint.PREFLIGHT_MS, deadline - Date.now())) });
+        // The candidate goes through the SAME checkpoint the resident getter uses: this is the one
+        // presentation that does not read the cache, so routing it here is what makes the refusal a
+        // property of every path rather than of the cached one. An already-dead re-signed generation is
+        // refused locally instead of spending a round trip to be told so, and — because this throws
+        // BEFORE the commit below — it can never become the resident connection's next-presented cred.
+        const proven = CotalEndpoint.presentableCreds(candidate, { renewable: true });
+        const probe = await probeConnect(this.servers, { creds: proven, tls: this.tls, timeoutMs: Math.max(500, Math.min(CotalEndpoint.PREFLIGHT_MS, deadline - Date.now())) });
         if (!probe.ok)
             throw new Error(`reloadCreds: the broker did not accept the re-signed credential (${probe.reason}); nothing adopted`);
         if (Date.now() > deadline)
@@ -694,11 +823,13 @@ export class CotalEndpoint extends EventEmitter {
             throw new Error(this.bearerSource
                 ? "this endpoint's user bearer has expired and renewal through the auth exchange is failing - not presenting the expired token to the broker; retrying with backoff"
                 : "this endpoint's user bearer has expired and it holds no bearer source to renew it - re-authenticate and rebuild the endpoint (construct it with a bearer FUNCTION for standing renewal)");
-        const credsExp = this.currentCreds && credsClaims(this.currentCreds).exp;
-        if (typeof credsExp === "number" && credsExp * 1000 <= Date.now())
-            throw new Error(this.credsSource
-                ? "this endpoint's creds have expired and renewal is failing - not presenting the expired credential to the broker; retrying with backoff"
-                : "this endpoint's creds have expired and it holds no creds source to renew them - replace the credential and rebuild the endpoint (pass a creds FUNCTION for standing renewal)");
+        // The CREDS refusal is NOT repeated here. It lives on the supply itself ({@link credsForWire}),
+        // which the authenticator below re-reads per attempt, so it covers this dial AND the reconnects
+        // nats.js runs on its own. Raising it early here too would only duplicate it on the one path
+        // that was already covered, and a fourth dial site added later would silently miss the copy.
+        // The refusal still surfaces on this path: the authenticator throws during the CONNECT, the
+        // library closes that attempt, and the reestablish loop's capped backoff re-enters here — where
+        // the source re-fetch above is what recovers a renewable endpoint.
         this.nc = await dialerFor(this.servers)({
             servers: this.servers,
             // In USER MODE the connection `name` carries the client-chosen inbox nonce (= connId) the callout
@@ -714,12 +845,32 @@ export class CotalEndpoint extends EventEmitter {
             inboxPrefix: `_INBOX_${this.connId}`,
             // The bearer rides a GETTER: nats.js re-evaluates the token authenticator per (re)connect
             // attempt, so internal reconnects present whatever refreshBearer last fetched.
-            // Creds likewise ride a GETTER when a source renews them, so internal reconnects (incl. the
-            // one the broker forces at JWT `exp`) present whatever refreshCreds last fetched.
-            ...authOpts({ token: this.token, user: this.user, pass: this.pass, creds: this.credsSource ? () => this.currentCreds : this.currentCreds, bearer: this.userMode ? () => this.currentBearer : undefined, sentinelCreds: this.sentinelCreds, tls: this.tls }),
+            // Creds ALWAYS ride the CHECKED getter, renewed or static, so every attempt — including the
+            // reconnects nats.js performs on its own (the one the broker forces at JWT `exp`, and a dial
+            // loop from an earlier drop that crosses `exp` mid-retry) — re-reads a credential that has
+            // just been proven unexpired rather than whatever the cache happens to hold.
+            // The gate is `!== undefined`, NOT truthiness. An EMPTY creds string is a caller that meant to
+            // authenticate and supplied nothing; on a truthiness gate it fell through to `creds: undefined`
+            // and dialed ANONYMOUSLY, so the broker answered `Authorization Violation` and the real fault
+            // (an empty credential) was never named. Routing it into the checked getter fails it loud
+            // instead. Anonymous access stays reachable the only way it should be: by passing no creds.
+            ...authOpts({ token: this.token, user: this.user, pass: this.pass, creds: this.currentCreds !== undefined || this.credsSource ? () => this.credsForWire() : undefined, bearer: this.userMode ? () => this.currentBearer : undefined, sentinelCreds: this.sentinelCreds, tls: this.tls }),
         });
+        this.armAuthExpiryReconnectFence(this.nc);
         this.watchStatus();
         this.js = jetstream(this.nc);
+        // Discovery of the issued generation (SPEC 13.15): the ISSUER wrote the accepted reference
+        // under the token this launch was handed; the broker admits the read only under this
+        // connection's own per-key grant. What comes back must name this incarnation, and a fresh
+        // connection re-reads it: the generation is bound to the credential the transport presented,
+        // never to a file that may have been replaced under it.
+        if (this.acceptedToken !== undefined) {
+            const ref = await readAcceptedRow(this.nc, this.space, this.acceptedToken);
+            const uid = this.requireLifecycleUid("an issued endpoint");
+            if (ref.owner !== this.owner || ref.actor !== this.actor || ref.uid !== uid)
+                throw new Error(`the accepted row names ${ref.owner}.${ref.actor} (uid ${ref.uid}), not this endpoint ${this.owner}.${this.actor} (uid ${uid}); refusing to ride a foreign issuance (SPEC 13.15)`);
+            this.issuedGeneration = ref.generation;
+        }
         if (this.doWatch || this.doRegister) {
             const kvm = new Kvm(this.nc);
             // The presence bucket is a JetStream stream. Open mode lazily creates it; auth mode
@@ -783,7 +934,20 @@ export class CotalEndpoint extends EventEmitter {
             }
         }
         if (this.doRegister) {
-            await this.publishPresence();
+            // #1356: this await is bounded already — a refused presence write returns no response at all,
+            // so it surfaces as the JetStream request TIMEOUT (~5s), once, with no retry here. What it did
+            // NOT do was say what failed: the bare `timeout` names no bucket, no space and no subsystem,
+            // and a launcher that catches it has nothing to act on. The lifecycle-proof failure a few lines
+            // above throws a fully explanatory message; this one is held to the same standard. Still fails
+            // rather than degrading — SPEC 13.1 fail-before-presence means a registering agent that cannot
+            // publish presence must not come up as though it had.
+            try {
+                await this.publishPresence();
+            }
+            catch (e) {
+                const detail = e?.message ?? String(e);
+                throw new Error(`presence registration failed for ${this.card.id}: the space's presence KV bucket "${presenceBucket(this.space)}" did not accept this endpoint's first write (${detail}). The bucket can be OPENED and WATCHED while refusing every write, so a healthy-looking connection does not rule this out; a broker whose store has latched refuses until it is restarted, and no client can clear it (#1356).`);
+            }
             this.heartbeatTimer = setInterval(() => {
                 this.publishPresence().catch((e) => this.emitRecoverable(e));
             }, this.heartbeatMs);
@@ -817,6 +981,10 @@ export class CotalEndpoint extends EventEmitter {
      *  second heartbeat, double-pump a consumer, or keep stale roster ghosts. Caller-owned
      *  subs (tap/serve) are left alone — they aren't rebuilt here. */
     clearConnectionScoped() {
+        if (this.authExpiryReconnectTimer) {
+            clearTimeout(this.authExpiryReconnectTimer);
+            this.authExpiryReconnectTimer = undefined;
+        }
         if (this.heartbeatTimer) {
             clearInterval(this.heartbeatTimer);
             this.heartbeatTimer = undefined;
@@ -834,6 +1002,8 @@ export class CotalEndpoint extends EventEmitter {
             }
         }
         this.streamMsgs.length = 0;
+        this.presenceEpoch++;
+        this.presenceRebind = undefined;
         try {
             this.presenceWatchIter?.stop();
         }
@@ -860,7 +1030,15 @@ export class CotalEndpoint extends EventEmitter {
         this.chatSubDenied.clear();
         this.confirmingChatSubs.clear();
         this.roster.clear();
+        // #1356: the presence-refusal record is connection-scoped like everything else torn down here.
+        // It says "the broker on THIS connection refuses writes to this bucket", so it cannot outlive the
+        // connection that observed it. Left behind, a later failure on a different connection inherits it
+        // and is reported as a bucket refusal — measured: a failed bind against an unreachable server
+        // still printed the refusal sentence while `connectionIssue` already said "connection refused".
+        this.clearPresenceWriteFailure();
         this.lastPresenceWatchAt = 0;
+        this.presenceRebindAt = 0;
+        this.presenceWatchEmpty = false;
         this.presenceSnapshotPopulated = false;
         this.emitPresenceViewIfChanged();
         this.joinSeq.clear();
@@ -894,6 +1072,8 @@ export class CotalEndpoint extends EventEmitter {
      * retry starts with the same empty state as a first attempt. */
     async closeFailedBind() {
         const failedNc = this.nc;
+        if (failedNc)
+            this.disableLibraryReconnect(failedNc);
         this.clearConnectionScoped();
         this.nc = undefined;
         this.js = undefined;
@@ -980,13 +1160,10 @@ export class CotalEndpoint extends EventEmitter {
         if (!this.stopped)
             return false;
         const nc = this.nc;
+        if (nc)
+            this.disableLibraryReconnect(nc);
         this.clearConnectionScoped();
-        try {
-            await nc?.drain();
-        }
-        catch {
-            /* already closing */
-        }
+        await this.closeWithoutLibraryReconnect(nc);
         this.nc = undefined;
         return true;
     }
@@ -1028,15 +1205,17 @@ export class CotalEndpoint extends EventEmitter {
         return p;
     }
     /** The transition: stop the connection-scoped timers FIRST (so nothing live touches
-     *  this.nc during the null window), drop the connection refs, drain the old nc, then
+     *  this.nc during the null window), drop the connection refs, close the old nc, then
      *  rebind + re-arm the supervisor on the fresh connection. clearConnectionScoped is
      *  idempotent, so connectAndBind's own call here is a noop. */
     async doRebuild() {
         const oldNc = this.nc;
+        if (oldNc)
+            this.disableLibraryReconnect(oldNc);
         this.reconnecting = true;
         try {
             this.clearConnectionScoped();
-            // Manual reconnect still has a live old epoch: complete broker-consumer cleanup before drain.
+            // Manual reconnect still has a live old epoch: complete broker-consumer cleanup before close.
             // Terminal self-heal has an already-closed epoch: disarm retains stream/name for fresh cleanup.
             if (oldNc && !oldNc.isClosed())
                 await Promise.all([...this.membershipFeedWatches].map((watch) => watch.arm));
@@ -1060,12 +1239,7 @@ export class CotalEndpoint extends EventEmitter {
             // the authoritative raw-liveness edge for the no-nc window until the new watcher seeds true.
             this.emit("transport", { connected: false });
             this.emit("connection", { connected: false });
-            try {
-                await oldNc?.drain();
-            }
-            catch {
-                /* already closing */
-            }
+            await this.closeWithoutLibraryReconnect(oldNc);
             await this.connectAndBind();
             // stop() may have run during the await — don't leave a live connection + heartbeat +
             // supervisor on a stopped endpoint. (Reads this.nc in its own scope — a bare `this.nc`
@@ -1149,10 +1323,17 @@ export class CotalEndpoint extends EventEmitter {
             throw e;
         }
     }
+    /** The presence epoch moves first: a bind still awaiting the broker must find it moved
+     *  before any await below gives it a window to install a watch on a stopped endpoint
+     *  (see {@link startPresenceWatch}). */
     async stop() {
         if (this.stopped)
             return;
+        if (this.nc)
+            this.disableLibraryReconnect(this.nc);
         this.stopped = true;
+        this.presenceEpoch++;
+        this.presenceRebind = undefined;
         // Wake a reestablishLoop sitting in backoff so it sees `stopped` and exits instead of
         // sleeping out retryMs; also clears the timer so it can't fire later.
         this.kickBackoff();
@@ -1164,6 +1345,8 @@ export class CotalEndpoint extends EventEmitter {
             clearTimeout(this.bearerTimer);
         if (this.credsTimer)
             clearTimeout(this.credsTimer);
+        if (this.authExpiryReconnectTimer)
+            clearTimeout(this.authExpiryReconnectTimer);
         for (const watch of this.membershipFeedWatches) {
             watch.stopped = true;
             watch.arm = watch.arm.catch(() => { }).then(async () => {
@@ -1215,11 +1398,17 @@ export class CotalEndpoint extends EventEmitter {
             /* best-effort graceful leave */
         }
         try {
-            await this.nc?.drain();
+            await this.closeWithoutLibraryReconnect(this.nc);
         }
         catch {
             /* ignore */
         }
+        // #1356: stop() does its own teardown and never runs clearConnectionScoped, so clear here too —
+        // AFTER the best-effort offline publish above, which would otherwise re-record a refusal on its
+        // way out. A stopped endpoint has no live basis for "this bucket is refusing writes", and the
+        // duration is computed from `Date.now()`, so a retained record does not merely go stale: it keeps
+        // COUNTING UP for as long as the dead object is held.
+        this.clearPresenceWriteFailure();
     }
     // ---- messaging -----------------------------------------------------------
     /** Multicast: broadcast to everyone on a channel. */
@@ -1550,7 +1739,12 @@ export class CotalEndpoint extends EventEmitter {
      *  launcher-supplied incarnation the rows are keyed on (ledger-consistent: the §13.1 presence
      *  lifecycle-proof refuses a divergent uid before any publish). */
     serviceCaller() {
-        return { owner: this.owner, actor: this.actor, uid: this.requireLifecycleUid("invokeService") };
+        const triple = { owner: this.owner, actor: this.actor, uid: this.requireLifecycleUid("invokeService") };
+        if (this.acceptedToken === undefined)
+            return triple;
+        if (this.issuedGeneration === undefined)
+            throw new Error("invokeService: the issued generation is not yet discovered on this connection");
+        return { ...triple, generation: this.issuedGeneration };
     }
     /** GENERIC v0.4 service invoke over this endpoint's own connection (P2 item 1, 1c.2b): resolve
      *  the named endpoint's registered surface — describe, §13.7 store fetch, digest-verified
@@ -1753,6 +1947,8 @@ export class CotalEndpoint extends EventEmitter {
             return { state: "current", fresh: true };
         if (!this.presenceSnapshotPopulated)
             return { state: "unpopulated", fresh: false };
+        if (this.presenceWatchEmpty)
+            return { state: "current", fresh: true };
         const staleSince = this.lastPresenceWatchAt + this.ttlMs;
         if (Date.now() < staleSince)
             return { state: "current", fresh: true };
@@ -2641,20 +2837,19 @@ export class CotalEndpoint extends EventEmitter {
                     delivered++;
                     if (m.seq >= ceiling) { // reached the page's upper bound
                         if (m.seq === ceiling) {
-                            try {
-                                out.push({ seq: m.seq, subject: m.subject, msg: m.json() });
-                            }
-                            catch { /* skip undecodable */ }
+                            const msg = historyMessageFromDelivery(m);
+                            if (msg)
+                                out.push({ seq: m.seq, subject: m.subject, msg });
                         }
                         complete = true;
                         break;
                     }
-                    try {
-                        out.push({ seq: m.seq, subject: m.subject, msg: m.json() });
+                    const msg = historyMessageFromDelivery(m);
+                    if (msg) {
+                        out.push({ seq: m.seq, subject: m.subject, msg });
                         if (out.length > limit)
                             out.shift();
                     }
-                    catch { /* skip undecodable */ }
                     if (delivered >= pending) {
                         complete = true;
                         break;
@@ -2709,6 +2904,7 @@ export class CotalEndpoint extends EventEmitter {
                     continue;
                 }
                 if (s.type === "reconnect") {
+                    this.armAuthExpiryReconnectFence(nc);
                     this.emit("transport", { connected: true, server: s.server });
                     continue;
                 }
@@ -3326,7 +3522,7 @@ export class CotalEndpoint extends EventEmitter {
     async startPlane3(aclFor, opts = {}) {
         if (!this.js)
             throw new Error("endpoint not started");
-        this.plane3 = { aclFor, reloadMembershipCreds: opts.reloadMembershipCreds, evictPrincipal: opts.evictPrincipal, planeConnLiveness: opts.planeConnLiveness, principalLiveness: opts.principalLiveness };
+        this.plane3 = { aclFor, reloadMembershipCreds: opts.reloadMembershipCreds, evictPrincipal: opts.evictPrincipal, planeConnLiveness: opts.planeConnLiveness, principalLiveness: opts.principalLiveness, reloadStoreIdentity: opts.reloadStoreIdentity };
         await this.armPlane3();
     }
     /** Serve one runtime durable-membership control request (the server-side delivery daemon). The caller
@@ -3666,6 +3862,21 @@ export class CotalEndpoint extends EventEmitter {
                 return { ok: false, error: e.message };
             }
         }
+        if (req.op === "reloadStoreIdentity") {
+            // Proof that this daemon reloads standing creds from ONE named store. The manager
+            // compares it to its remint store before every renewal pass. Absent hook
+            // is a daemon that cannot name its store, which is itself a divergent composition.
+            if (!this.plane3?.reloadStoreIdentity)
+                return { ok: false, error: "reloadStoreIdentity: this daemon did not name the SecretStore it reloads from" };
+            try {
+                const identity = this.plane3.reloadStoreIdentity();
+                // Round-trip through the closed parser so a hook cannot smuggle extra fields onto the rail.
+                return { ok: true, data: parseSecretStoreIdentity(identity) };
+            }
+            catch (e) {
+                return { ok: false, error: e.message };
+            }
+        }
         return { ok: false, error: `op "${req.op}" not supported on the delivery admin service` };
     }
     /** Fan-out loop: bind the privileged `fanout` durable on CHAT and route each message (routing only —
@@ -4605,18 +4816,99 @@ export class CotalEndpoint extends EventEmitter {
         // the publisher — this covers stop(), setStatus("offline"), and any future offline publish site, so
         // the raw KV record is compliant, not only the observer-side roster materialization.
         const record = this.status === "offline" ? this.toOffline(p) : p;
-        await this.kv.put(this.card.id, JSON.stringify(record));
+        // #1356: this put can still be in flight when a teardown runs, because a rebind reaches
+        // publishPresence through onPresenceBucketEmpty and no teardown awaits that flight. Take the same
+        // epoch fence {@link startPresenceWatch} takes, so a put that outlives its epoch still throws to
+        // its caller but no longer writes presence-refusal state belonging to a later connection. Without
+        // it, a heartbeat put (default 2s) whose ~5s JetStream timeout elapses after a rebuild plants a
+        // refusal on the connection that just published successfully.
+        const epoch = this.presenceEpoch;
+        try {
+            await this.kv.put(this.card.id, JSON.stringify(record));
+        }
+        catch (e) {
+            // #1356: a broker can put this bucket into a state where it refuses every write and never
+            // recovers, and the write is the ONLY thing that fails — open and watch both still succeed, so
+            // nothing else here notices. Record WHEN the refusals started, at the one site that knows the
+            // failing write was a presence write; a caller cannot infer that from the generic `warning`
+            // stream, which carries any recoverable error.
+            if (epoch === this.presenceEpoch && !this.stopped) {
+                this.presenceWriteFailingSince ??= Date.now();
+                this.lastPresenceWriteError = e?.message ?? String(e);
+            }
+            throw e;
+        }
+        // A late SUCCESS is the same hazard mirrored, and this fence answers only the cross-epoch half
+        // of it: a success belonging to a retired epoch cannot erase a refusal the current one
+        // established from its own evidence. It does NOT order puts within a single epoch, because it
+        // compares epoch identity rather than which put is the latest evidence, so an earlier put that
+        // succeeds late still clears a later put's refusal. Heartbeats run at 2s against a ~5s put
+        // timeout, so that overlap is routine rather than a corner, and the next failing put re-plants
+        // the record with a fresh `since`. Tracked in #1461, not repaired here.
+        if (epoch !== this.presenceEpoch || this.stopped)
+            return;
+        this.clearPresenceWriteFailure();
     }
+    /** #1356: drop the presence-refusal record when the connection that OBSERVED those refusals goes
+     *  away. "This bucket is refusing writes" is a claim about a specific broker connection; once that
+     *  connection is torn down or rebuilt the claim has no remaining basis, and a later failure on a
+     *  fresh connection must establish it again from its own evidence.
+     *
+     *  Cleared at the SOURCE rather than guarded at each reader, because a guard protects one consumer
+     *  and clearing protects every consumer, including ones not yet written. Measured: without this, a
+     *  failed bind against an unreachable server still reported the presence-refusal sentence while the
+     *  endpoint's own `connectionIssue` already said "connection refused". */
+    clearPresenceWriteFailure() {
+        this.presenceWriteFailingSince = undefined;
+        this.lastPresenceWriteError = undefined;
+    }
+    /** #1356: the presence bucket has been refusing writes since this time, or `undefined` when the
+     *  last publish succeeded. Cleared by the first successful write, so a survived blip reads as
+     *  healthy and only a SUSTAINED failure carries a duration.
+     *
+     *  Presence writes are the CANARY, not the scope: the broker can disable JetStream account-wide
+     *  while the NATS connection stays up, so a caller must not read this as "only presence is
+     *  affected". It reports what was observed, not how far the fault extends. */
+    presenceWriteFailure() {
+        if (this.presenceWriteFailingSince === undefined)
+            return undefined;
+        return {
+            since: this.presenceWriteFailingSince,
+            forMs: Date.now() - this.presenceWriteFailingSince,
+            error: this.lastPresenceWriteError,
+            bucket: presenceBucket(this.space),
+        };
+    }
+    /** Bind a presence watch on the current connection. Resolves true when the watch was
+     *  installed, false when the endpoint stopped or rebuilt while the bind was in flight: that
+     *  bind's iterator is released here and nothing is installed, because the epoch that asked
+     *  for it is gone and the epoch that replaced it binds its own watch through
+     *  {@link connectAndBind}. Without this fence a bind that completes after {@link stop} would
+     *  resurrect a watch on a stopped endpoint, and one that completes after a rebuild would
+     *  overwrite the fresh epoch's watch with a dead-connection iterator. */
     async startPresenceWatch() {
         if (!this.kv)
-            return;
+            return false;
+        const epoch = this.presenceEpoch;
         let hydrated;
         this.presenceSnapshot = new Promise((resolve) => { hydrated = resolve; });
         const iter = await this.kv.watch();
+        if (epoch !== this.presenceEpoch) {
+            try {
+                iter.stop();
+            }
+            catch { /* its connection may already be gone */ }
+            hydrated();
+            return false;
+        }
         this.presenceWatchIter = iter;
         void (async () => {
             let ready = false;
             for await (const e of iter) {
+                // A rebind bound a newer watch: this one's remaining entries belong to a retired epoch.
+                // Stop consuming them (the rebind stops the iterator too; this ends the loop first).
+                if (this.presenceWatchIter !== iter)
+                    break;
                 this.handleKvEntry(e);
                 // @nats-io/kv marks the final initial replay entry isUpdate=true. Later updates stay true.
                 if (!ready && e.isUpdate) {
@@ -4628,6 +4920,68 @@ export class CotalEndpoint extends EventEmitter {
             }
             hydrated();
         })().catch((e) => this.emit("error", e));
+        return true;
+    }
+    /**
+     * Replace a presence watch that has gone silent past TTL while the connection is up. The new
+     * ordered consumer starts from the bucket's current last-per-subject state, so a peer that is
+     * heartbeating is re-observed within one replay and a peer that is gone is aged out by the
+     * next sweep exactly as if the watch had never stalled. Rate-limited to one attempt per TTL
+     * per observer, never overlapping, never on a stopped or rebuilding endpoint (those own their
+     * watch through {@link connectAndBind}). A stop or rebuild that lands while the bind is in
+     * flight retires it: {@link startPresenceWatch} releases the late iterator and reports
+     * nothing, since the epoch that was silent no longer exists. A failed bind is reported and
+     * the view stays stale.
+     */
+    rebindStalePresenceWatch(now) {
+        if (this.stopped || this.reconnecting || !this.kv || !this.nc || this.nc.isClosed())
+            return;
+        if (this.presenceRebind || now - this.presenceRebindAt < this.ttlMs)
+            return;
+        this.presenceRebindAt = now;
+        const old = this.presenceWatchIter;
+        const epoch = this.presenceEpoch;
+        this.presenceRebind = (async () => {
+            try {
+                const silentMs = now - this.lastPresenceWatchAt;
+                // Bind the replacement FIRST. A rebind the broker refuses (stream gone, create denied) or
+                // one a held link never answers must leave the old watch in place: on a plain stall that
+                // watch is the one that recovers by itself, and its replay is still guarded against
+                // expired PUTs. Only a successfully bound watch retires its predecessor.
+                const installed = await this.startPresenceWatch();
+                // Retired mid-bind (stop or rebuild moved the epoch): the late iterator is already
+                // released and the old watch was torn down by whoever moved the epoch. Nothing to
+                // retire, nothing to report.
+                if (!installed)
+                    return;
+                if (old && old !== this.presenceWatchIter) {
+                    try {
+                        old.stop();
+                    }
+                    catch { /* already closed with its consumer */ }
+                }
+                // A bucket with no keys replays nothing, so the new watch cannot refresh
+                // `lastPresenceWatchAt` by delivering. It IS current knowledge: nobody is present. Read
+                // the consumer's initial pending count for that one fact; nats.js's KV watch computed it
+                // from the same `info(true)` it used to place the isUpdate marker.
+                const pending = this.presenceWatchIter?._data?._info?.num_pending;
+                if (pending === 0)
+                    await this.onPresenceBucketEmpty();
+                this.emit("warning", new Error(`presence watch silent for ${silentMs}ms with the connection up; rebound it from the bucket's current state`));
+            }
+            catch (e) {
+                // A bind the epoch swap itself rejected (connection drained under it) is not a fault of
+                // the epoch that replaced it; only a refusal on a still-current epoch is reported.
+                if (epoch === this.presenceEpoch)
+                    this.emit("error", e);
+            }
+            finally {
+                // An epoch swap already disowned this flight (and may own a successor's by now); only
+                // a flight still in its own epoch clears the slot. Within one epoch there is one flight.
+                if (epoch === this.presenceEpoch)
+                    this.presenceRebind = undefined;
+            }
+        })();
     }
     /** Watch the channel registry: replay existing keys, then stream updates, into the local
      *  cache. Best-effort — a registry the endpoint can't read leaves the cache empty (effective
@@ -4667,8 +5021,65 @@ export class CotalEndpoint extends EventEmitter {
             /* keep last good */
         }
     }
+    /** The watch was just bound onto a bucket with no keys.
+     *
+     *  A NON-REGISTERING observer (a `cotal status` probe, a lease checker) has real knowledge:
+     *  nobody is present. Every peer still in its roster is known gone (its key is not there to
+     *  replay), so it is marked offline now rather than aged out against a delivery that cannot
+     *  come; the silence gate is disarmed and the view reads current until the first write lands.
+     *  Without that the empty-bucket view relapsed to stale one window later and rebound again on
+     *  every window, one consumer create and one warning per TTL for as long as the mesh was empty.
+     *
+     *  A REGISTERING observer (the manager) is itself one of the keys that should be there. An
+     *  empty bucket under it means the bucket was wiped since its last heartbeat (the netcup
+     *  recreation), and the same wipe took every peer's record: their absence says the bucket is
+     *  new, not that they left. rev-1421-gpt reproduced the previous behaviour at default timing:
+     *  the rebind landed ~0.9s after the recreation, the observer marked every peer AND ITSELF
+     *  offline, and held the view current for up to one heartbeat, a false verdict `cotal ps`
+     *  would print as `mesh offline`. So a registering observer re-publishes its own record NOW,
+     *  which the new watch delivers, and lets the ordinary per-peer age-out run from that delivery:
+     *  a peer that is still heartbeating rewrites its key within its own heartbeat interval and is
+     *  re-observed live; one that is gone ages out exactly as after a plain rebind. The roster is
+     *  not touched here and the view is not held; the delivery is what makes it current.
+     *
+     *  What neither branch covers: a consumer that dies again while the bucket is still empty is
+     *  not detectable by silence, so the first write after that is missed until the observer
+     *  restarts. */
+    async onPresenceBucketEmpty() {
+        if (this.doRegister) {
+            // Our own key is missing from a bucket we write to: put it back. publishPresence throws on
+            // a refused write; that surfaces through the rebind flight's error path with the epoch
+            // fence intact, and the view stays stale, which is the honest state for an observer that
+            // cannot even record itself.
+            await this.publishPresence();
+            return;
+        }
+        this.markPresenceBucketEmpty();
+    }
+    /** See {@link onPresenceBucketEmpty}: the non-registering branch. */
+    markPresenceBucketEmpty() {
+        this.presenceWatchEmpty = true;
+        this.lastPresenceWatchAt = 0;
+        let changed = false;
+        for (const [id, p] of this.roster) {
+            if (p.status === "offline")
+                continue;
+            const offline = this.toOffline(p);
+            this.roster.set(id, offline);
+            this.emit("presence", { type: "offline", presence: offline });
+            changed = true;
+        }
+        if (changed)
+            this.emit("roster", this.getRoster());
+        this.emitPresenceViewIfChanged();
+    }
     handleKvEntry(e) {
+        // Provisioning uses this one reserved key to prove the bucket's max_age is enforced (#404).
+        // It is maintenance traffic, never a peer identity, an offline transition, or view freshness.
+        if (e.key === TTL_RECONCILE_CANARY_KEY)
+            return;
         this.lastPresenceWatchAt = Date.now();
+        this.presenceWatchEmpty = false;
         if (e.operation === "DEL" || e.operation === "PURGE") {
             this.markOffline(e.key);
             return;
@@ -4774,11 +5185,29 @@ export class CotalEndpoint extends EventEmitter {
         // watch freshness; surface the view as stale instead.
         if (this.lastPresenceWatchAt !== 0 && now - this.lastPresenceWatchAt > this.ttlMs) {
             this.emitPresenceViewIfChanged();
+            // Staying stale is the right verdict for a held link (#1045), and the wrong END STATE when
+            // the transport is up and the watch's own consumer is what died. Measured on netcup
+            // 2026-09-09: the presence stream was deleted and recreated, its sequence restarted, and
+            // every observer's ORDERED consumer re-created itself at the OLD start sequence (nats.js
+            // 3.4.0 resets from its cursor). The broker kept sending idle heartbeats, so the client
+            // never reset again, the iterator never closed, and the manager's roster stayed frozen at
+            // the pre-recreation snapshot for hours: `cotal ps` read every older seat "mesh offline"
+            // and every newer seat "not in roster" while all of them were heartbeating. The same
+            // end state follows a plain consumer delete (an operator, or the 5-minute inactive
+            // threshold after a long stall). Rebind the watch from the bucket's CURRENT state; a held
+            // link's rebind fails or stays silent and the view simply stays stale, as before.
+            this.rebindStalePresenceWatch(now);
             return;
         }
         let changed = false;
         for (const [id, p] of this.roster) {
-            if (p.status !== "offline" && now - p.ts > this.ttlMs) {
+            // A peer's own `ts` always trails the observer's last delivery, so "older than TTL by the
+            // wall clock" alone ages peers out on the tick just before the whole-bucket gate above
+            // trips (#1311's flap; measured as 3 offline verdicts per silence in the rebind suite).
+            // Require that the watch itself delivered for a full TTL after this peer's last heartbeat:
+            // then other peers were heard and this one was not, which is the only silence that is the
+            // peer's rather than the observer's.
+            if (p.status !== "offline" && now - p.ts > this.ttlMs && this.lastPresenceWatchAt - p.ts > this.ttlMs) {
                 const offline = this.toOffline(p);
                 this.roster.set(id, offline);
                 this.emit("presence", { type: "offline", presence: offline });
@@ -4817,10 +5246,18 @@ function kindFromParsed(kind) {
             throw new Error(`cannot derive a message kind from subject kind "${kind}"`);
     }
 }
-/** Routing fields in the envelope are advisory. Surface a channel label only from the authenticated
- * chat subject, so connector attention cannot be bypassed with a mismatched payload `channel`. */
+/** Routing fields in the envelope are advisory. The broker forge-locks sender (and for DMs,
+ *  recipient) into the subject. Callers MUST already have rejected a missing `from`, an
+ *  unparseable subject, or `from.id !== parsed.sender` (SPEC §5). This derives the remaining
+ *  routing tokens from the subject for rows that survived — it does not rewrite a mismatched
+ *  `from.id`. Live tails, channel backfill, and channel recall skip the mismatch; history
+ *  does the same (#388). */
 function authenticatedMessage(msg, parsed) {
-    return parsed.kind === "chat" ? authenticatedChannelMessage(msg, parsed.rest) : msg;
+    if (parsed.kind === "chat")
+        return authenticatedChannelMessage(msg, parsed.rest);
+    if (parsed.kind === "inst")
+        return authenticatedDmMessage(msg, parsed.rest);
+    return msg;
 }
 function authenticatedChannelMessage(msg, channel) {
     if (msg.channel === channel && msg.to === undefined && msg.toService === undefined)
@@ -4828,6 +5265,49 @@ function authenticatedChannelMessage(msg, channel) {
     const { to: _to, toService: _toService, ...base } = msg;
     return { ...base, channel };
 }
+function authenticatedDmMessage(msg, to) {
+    if (msg.to === to && msg.channel === undefined && msg.toService === undefined)
+        return msg;
+    const { channel: _channel, toService: _toService, ...base } = msg;
+    return { ...base, to };
+}
+/** History drain keeps `m.json()` and used to throw the subject away. SPEC §5: on receive, verify
+ *  `from.id` equals the subject sender; on mismatch, a missing `from`, or an unparseable delivery
+ *  subject, reject and never surface. Fail closed on shape too: a stored JSON `null` or a truthy
+ *  non-object `from` must not throw mid-array. Do not echo-drop `from.id === this.card.id`:
+ *  god-view history must include the viewer's own sends. */
+function historyMessageFromDelivery(m) {
+    let raw;
+    try {
+        raw = m.json();
+    }
+    catch {
+        return undefined;
+    }
+    if (!isHistoryDrainEnvelope(raw))
+        return undefined;
+    const parsed = parseSubject(m.subject);
+    if (!parsed || !isPrincipalOwnerToken(parsed.owner))
+        return undefined;
+    if (raw.from.id !== parsed.sender)
+        return undefined;
+    return authenticatedMessage(raw, parsed);
+}
+/**
+ * Narrow enough for the type checker and for fail-closed history: object envelope, usable `id`,
+ * object `from`. That is what lets `from.id !== parsed.sender` run without throwing, and what
+ * lets `authenticatedMessage` take the row without a cast.
+ *
+ * Does NOT verify SPEC §5 message shape. It does not require a string `from.id` (the SPEC §5
+ * comparison still rejects a mismatch), exactly one route key, a finite `ts`, a string
+ * `space`, a full EndpointRef `from` (`name`/`role`), or well-formed `parts`. Those belong
+ * to `isCotalMessage` (Plane-3). History must not use that guard: a public
+ * `unicast(..., { parts: [{ kind: "data", data: undefined }] })` serializes to `{kind:"data"}`
+ * and must still surface.
+ */
+function isHistoryDrainEnvelope(value) {
+    return isRecord(value) && isUsableMessageId(value.id) && isRecord(value.from);
+}
 function isPlane3DeliveryFrame(value) {
     if (!isRecord(value))
         return false;