@cotal-ai/core 0.101.6 → 0.102.0

package/dist/endpoint.js

@@ -7,6 +7,7 @@ import { wsconnect } from "@nats-io/nats-core";
 import { parseLivenessAnswer, responderFromLease, responderFromProbe, isLivenessPlane, LIVENESS_PLANES, } from "./liveness.js";
 import { processRuntime } from "./process-runtime.js";
 import { credsClaims, credsFingerprint, credsRenewalDelayMs, idFromCreds } from "./identity.js";
+import { armRenewal, renewalDeadlineMs, jitteredBackoffMs } from "./renewal.js";
 import { requireBrokerFloor } from "./broker-floor.js";
 import { withDeadline } from "./deadline.js";
 import { inspectCredHealth } from "./provision.js";
@@ -180,7 +181,15 @@ export class CotalEndpoint extends EventEmitter {
     bearerSource;
     /** The freshest bearer — what every (re)connect attempt presents. */
     currentBearer;
-    bearerTimer;
+    /** Cancels the armed bearer renewal or retry. */
+    cancelBearerRenewal;
+    /** Consecutive failed bearer renewals: the retry's step, and whether this run of failures has
+     *  already been surfaced. */
+    bearerRenewalFailures = 0;
+    /** Whether the expired-bearer refusal has been surfaced for this outage. */
+    bearerRefusalSurfaced = false;
+    /** When the held bearer was adopted: its lifetime's start when it carries no `iat`. */
+    bearerAdoptedAtMs = 0;
     /** Arms against the credential authenticated on the current wire. nats-core 3.4.0 discards the
      *  promise returned by its async transport-close continuation; if that continuation enters the
      *  reconnect dial loop with an expired JWT, its terminal auth error becomes an unhandled rejection.
@@ -659,11 +668,12 @@ export class CotalEndpoint extends EventEmitter {
             return;
         this.superviseConnection();
     }
-    /** How far ahead of the current bearer's `exp` a refresh fires, and how soon a FAILED refresh
-     *  retries. The margin must clear a reconnect window (nats.js retries use the sync token getter,
-     *  so whatever `currentBearer` holds is what every attempt presents). */
-    static BEARER_REFRESH_MARGIN_MS = 60_000;
-    static BEARER_RETRY_MS = 15_000;
+    /** The floor on a bearer renewal's delay. A token whose whole lifetime sits under it is renewed at
+     *  this cadence (`expiry-renewal` mints 5s bearers). */
+    static BEARER_MIN_RENEWAL_MS = 5_000;
+    /** A failed bearer renewal retries after about this long, doubling per failure up to the cap. */
+    static BEARER_RETRY_MS = 5_000;
+    static BEARER_RETRY_CAP_MS = 60_000;
     /**
      * A condition the endpoint is already surviving. Node rethrows `error` when no listener is
      * attached, so emitting retry notices on `error` killed hosts that the endpoint intended to
@@ -672,10 +682,11 @@ export class CotalEndpoint extends EventEmitter {
     emitRecoverable(err) {
         this.emit("warning", err);
     }
-    /** Fetch a fresh bearer from the source, pin its principal to ours, arm the next refresh. On a
+    /** Fetch a fresh bearer from the source, pin its principal to ours, arm the next renewal. On a
      *  fetch/principal failure: THROWS when `initial` (start() must fail loud before first connect);
-     *  otherwise emits "warning" and retries — the live connection keeps working until its current JWT
-     *  expiry, so a dead auth service is loud without instantly dropping the mesh. */
+     *  otherwise retries on a jittered, capped backoff, and warns once for the whole run of failures —
+     *  the live connection keeps working until its current JWT expiry, so a dead auth service is
+     *  loud without instantly dropping the mesh, and without a line per attempt. */
     async refreshBearer(initial = false) {
         try {
             const bearer = await this.bearerSource();
@@ -698,11 +709,10 @@ export class CotalEndpoint extends EventEmitter {
                 // {@link presentableCreds} (#1572).
                 if (expiryMs <= Date.now())
                     throw new Error("the bearer source returned a token that has already expired - nothing adopted");
-                // THE SAME BYTES BACK AGAIN from a source whose token cannot carry the next cycle: the
-                // delay it arms is non-positive, `armBearerRefresh` floors that to 5s, and the next read
-                // returns that same token - a 5s loop against the auth service for the rest of its life,
-                // with BEARER_RETRY_MS bypassed because the fetch did not FAIL. It succeeded and returned
-                // nothing new (#1561).
+                // THE SAME BYTES BACK AGAIN from a source whose token is already past its renewal window:
+                // the renewal it arms is floored to 5s, and the next read returns that same token - a 5s
+                // loop against the auth service for the rest of its life, with the retry backoff bypassed
+                // because the fetch did not FAIL. It succeeded and returned nothing new (#1561).
                 //
                 // The test is byte identity, not expiry, and not advancement. `exp` carries one second of
                 // resolution, so a key rotation re-signing the same claims under a new key - and a healthy
@@ -710,30 +720,63 @@ export class CotalEndpoint extends EventEmitter {
                 // `exp` has not moved. That is genuinely re-issued material, which an advancement test
                 // refuses and this one adopts. It is the question the creds path asks with
                 // `credsFingerprint`: did the source re-issue ANYTHING (#1572).
-                if (bearer === held && expiryMs - Date.now() - CotalEndpoint.BEARER_REFRESH_MARGIN_MS <= 0)
+                if (bearer === held && Date.now() >= bearerRenewalDeadlineMs(held, this.bearerAdoptedAtMs))
                     throw new Error("the bearer source re-served the token already held and it cannot carry another cycle (the auth service has not issued a fresh one) - nothing adopted");
             }
+            if (bearer !== held)
+                this.bearerAdoptedAtMs = Date.now();
             this.currentBearer = bearer;
-            // A non-positive delay is NOT clamped to BEARER_RETRY_MS. A deployment whose whole token TTL
-            // sits inside the margin is legitimate - `expiry-renewal` mints 5s bearers against the 60s
-            // margin - and for it the 5s floor IS the renewal cadence. Backing that source off to 15s
-            // leaves a 5s token dead for two thirds of every cycle. What made #1561 a pointless loop was
-            // the source returning nothing new, which is refused above, not the cadence itself.
-            this.armBearerRefresh(expiryMs - Date.now() - CotalEndpoint.BEARER_REFRESH_MARGIN_MS);
+            this.bearerRenewalFailures = 0;
+            // The renewal point is drawn from the middle of the token's lifetime, spaced from this
+            // process's other renewals in the space: a fixed margin before `exp` kept every endpoint
+            // issued in one second renewing, and rebinding through the callout, in one second for life.
+            const { issuedAtMs, expiresAtMs } = bearerLifetimeMs(bearer, this.bearerAdoptedAtMs);
+            this.armBearerTimer(armRenewal(this.space, issuedAtMs, expiresAtMs, () => void this.renewBearerOnTimer(), CotalEndpoint.BEARER_MIN_RENEWAL_MS));
         }
         catch (e) {
             if (initial)
                 throw e;
-            this.emitRecoverable(new Error(`bearer refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current token's expiry if the auth service stays down`));
-            this.armBearerRefresh(CotalEndpoint.BEARER_RETRY_MS);
+            if (this.bearerRenewalFailures === 0)
+                this.emitRecoverable(new Error(`bearer refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying with backoff; this connection dies at its current token's expiry if the auth service stays down`));
+            // A wait the source named is kept as named: never retried inside it, nor stretched by the backoff.
+            const named = retryAfterMs(e);
+            const delay = named > 0 ? named : jitteredBackoffMs(this.bearerRenewalFailures, CotalEndpoint.BEARER_RETRY_MS, CotalEndpoint.BEARER_RETRY_CAP_MS);
+            this.bearerRenewalFailures++;
+            const timer = setTimeout(() => void this.renewBearerOnTimer(), delay);
+            timer.unref?.();
+            this.armBearerTimer(() => clearTimeout(timer));
         }
     }
-    armBearerRefresh(delayMs) {
+    armBearerTimer(cancel) {
+        this.cancelBearerRenewal?.();
+        this.cancelBearerRenewal = cancel;
         if (this.stopped)
+            cancel();
+    }
+    /** The timed renewal. A new token moves the live connection onto itself at once, through the
+     *  same controlled reconnect the creds path uses, so the broker never has to expire a connection
+     *  this endpoint could renew. With no live connection, it ends the rebuild loop's wait instead:
+     *  that loop refuses to dial an expired token and leaves the retries to this timer. */
+    async renewBearerOnTimer() {
+        const held = this.currentBearer;
+        await this.refreshBearer();
+        if (this.stopped || this.currentBearer === held)
             return;
-        clearTimeout(this.bearerTimer);
-        this.bearerTimer = setTimeout(() => void this.refreshBearer(), Math.max(5_000, delayMs));
-        this.bearerTimer.unref?.();
+        if (this.nc && !this.nc.isClosed())
+            await this.swapConnectionOntoFreshCreds();
+        else
+            this.kickBackoff();
+    }
+    /** The cached bearer, checked: every (re)connect attempt reads it, the reconnects nats.js runs
+     *  on its own included, so none presents an expired token, which the callout could only deny
+     *  after a full round trip. The bearer counterpart of {@link credsForWire}. */
+    bearerForWire() {
+        const bearer = this.currentBearer;
+        if (bearer !== undefined && bearerExpiryMs(bearer) > Date.now())
+            return bearer;
+        throw new Error(this.bearerSource
+            ? "this endpoint's user bearer has expired and renewal through the auth exchange is failing - not presenting the expired token to the broker; retrying with backoff"
+            : "this endpoint's user bearer has expired and it holds no bearer source to renew it - re-authenticate and rebuild the endpoint (construct it with a bearer FUNCTION for standing renewal)");
     }
     /** How soon a FAILED creds refresh retries. Successful refreshes schedule by lifetime fraction
      *  (75% of iat→exp), not a fixed margin — standing creds span hours to days, bearers minutes. */
@@ -780,6 +823,12 @@ export class CotalEndpoint extends EventEmitter {
                 : "this endpoint was constructed with an empty creds string - not dialing without auth material (an empty credential is not anonymous access)");
         return CotalEndpoint.presentableCreds(this.currentCreds, { renewable: Boolean(this.credsSource) });
     }
+    /** The credential this endpoint presents on its next connect, checked as {@link credsForWire}
+     *  checks it. A second connection under the same identity reads it on every attempt, so it
+     *  presents each credential this endpoint adopts and never an expired one. */
+    wireCreds() {
+        return this.credsForWire();
+    }
     /** Refuses a fetched generation there is nothing to renew FROM, so the caller's failure posture
      *  ({@link CREDS_RETRY_MS}) applies instead of the renewal schedule (issue #1523).
      *
@@ -850,11 +899,11 @@ export class CotalEndpoint extends EventEmitter {
         const { iat, exp } = credsClaims(creds);
         return { iat, exp };
     }
-    /** Swap the live connection onto the freshest cached cred with a controlled `nc.reconnect()`
-     *  (nats.js re-evaluates the creds getter per attempt). Swapping now, instead of waiting for the
-     *  broker to close the connection at `exp`, means the wire never carries a near-dead JWT and the
-     *  operator never sees a spurious "authentication expired". After the reconnect succeeds, re-arm
-     *  the expiry fence for the credential now authenticated on the wire. */
+    /** Swap the live connection onto the freshest cached cred or bearer with a controlled
+     *  `nc.reconnect()` (nats.js re-evaluates the authenticator per attempt). Swapping now, instead of
+     *  waiting for the broker to close the connection at `exp`, means the wire never carries a
+     *  near-dead JWT and the operator never sees a spurious "authentication expired". After the
+     *  reconnect succeeds, re-arm the expiry fence for the credential now authenticated on the wire. */
     async swapConnectionOntoFreshCreds() {
         const nc = this.nc;
         if (nc && !this.stopped) {
@@ -1088,14 +1137,15 @@ export class CotalEndpoint extends EventEmitter {
     }
     async bindConnection() {
         this.clearConnectionScoped();
-        // Bearer-source endpoints fetch before the FIRST connect, and re-fetch on a rebuild whose
-        // cached token is already inside the refresh margin (a rebuild after a long outage would
-        // otherwise present a dead bearer for its first attempts).
+        // Bearer-source endpoints fetch before the FIRST connect, and renew first on a rebuild whose
+        // cached token is past its renewal window (a rebuild after a long outage would otherwise
+        // present a dead bearer). While renewals are failing, their retry timer owns the source: a
+        // fetch here too would add one call to the exchange per rebuild attempt.
         if (this.bearerSource) {
-            const stale = !this.currentBearer ||
-                bearerExpiryMs(this.currentBearer) - Date.now() < CotalEndpoint.BEARER_REFRESH_MARGIN_MS;
-            if (stale)
-                await this.refreshBearer(!this.currentBearer);
+            if (!this.currentBearer)
+                await this.refreshBearer(true);
+            else if (this.bearerRenewalFailures === 0 && Date.now() >= bearerRenewalDeadlineMs(this.currentBearer, this.bearerAdoptedAtMs))
+                await this.refreshBearer();
         }
         // Creds-source endpoints likewise fetch before the FIRST connect, and re-fetch on a rebuild
         // whose cached cred is expired or inside its renewal window.
@@ -1109,13 +1159,22 @@ export class CotalEndpoint extends EventEmitter {
         // instantly drop a still-live mesh), which used to leave the connect below presenting an
         // EXPIRED bearer: a guaranteed denial that still costs a full auth-callout round trip, repeated
         // by the reestablish loop at a flat rate for as long as the process lives. Refuse here instead.
-        // The loop's capped backoff paces the retries and re-enters this method, so a renewal that
-        // starts working reconnects on its own; an endpoint with no bearer source cannot renew at all,
-        // and says so.
-        if (this.userMode && this.currentBearer && bearerExpiryMs(this.currentBearer) <= Date.now())
-            throw new Error(this.bearerSource
-                ? "this endpoint's user bearer has expired and renewal through the auth exchange is failing - not presenting the expired token to the broker; retrying with backoff"
-                : "this endpoint's user bearer has expired and it holds no bearer source to renew it - re-authenticate and rebuild the endpoint (construct it with a bearer FUNCTION for standing renewal)");
+        // The loop's capped backoff paces the retries and re-enters this method, and a renewal that
+        // starts working ends its wait; an endpoint with no bearer source cannot renew at all, and says
+        // so. The refusal is surfaced once per outage: the loop retries a repeat without a notice.
+        if (this.userMode) {
+            try {
+                this.bearerForWire();
+                this.bearerRefusalSurfaced = false;
+            }
+            catch (e) {
+                if (!this.bearerRefusalSurfaced) {
+                    this.bearerRefusalSurfaced = true;
+                    throw e;
+                }
+                throw new RepeatedRefusal(e.message);
+            }
+        }
         // The CREDS refusal is NOT repeated here. It lives on the supply itself ({@link credsForWire}),
         // which the authenticator below re-reads per attempt, so it covers this dial AND the reconnects
         // nats.js runs on its own. Raising it early here too would only duplicate it on the one path
@@ -1139,8 +1198,9 @@ export class CotalEndpoint extends EventEmitter {
             ...(this.transportPingIntervalMs === undefined ? {} : { pingInterval: this.transportPingIntervalMs }),
             ...(this.transportMaxPingOut === undefined ? {} : { maxPingOut: this.transportMaxPingOut }),
             ...(this.transportConnectTimeoutMs === undefined ? {} : { timeout: this.transportConnectTimeoutMs }),
-            // The bearer rides a GETTER: nats.js re-evaluates the token authenticator per (re)connect
-            // attempt, so internal reconnects present whatever refreshBearer last fetched.
+            // The bearer rides the CHECKED getter: nats.js re-evaluates the token authenticator per
+            // (re)connect attempt, so internal reconnects present whatever refreshBearer last fetched,
+            // and refuse a token that has expired since, as the creds getter does.
             // Creds ALWAYS ride the CHECKED getter, renewed or static, so every attempt — including the
             // reconnects nats.js performs on its own (the one the broker forces at JWT `exp`, and a dial
             // loop from an earlier drop that crosses `exp` mid-retry) — re-reads a credential that has
@@ -1150,7 +1210,7 @@ export class CotalEndpoint extends EventEmitter {
             // and dialed ANONYMOUSLY, so the broker answered `Authorization Violation` and the real fault
             // (an empty credential) was never named. Routing it into the checked getter fails it loud
             // instead. Anonymous access stays reachable the only way it should be: by passing no creds.
-            ...authOpts({ token: this.token, user: this.user, pass: this.pass, creds: this.currentCreds !== undefined || this.credsSource ? () => this.credsForWire() : undefined, bearer: this.userMode ? () => this.currentBearer : undefined, sentinelCreds: this.sentinelCreds, tls: this.tls }),
+            ...authOpts({ token: this.token, user: this.user, pass: this.pass, creds: this.currentCreds !== undefined || this.credsSource ? () => this.credsForWire() : undefined, bearer: this.userMode ? () => this.bearerForWire() : undefined, sentinelCreds: this.sentinelCreds, tls: this.tls }),
         });
         // SPEC §13.12: the control surface requires nats-server >= 2.12; this runs on every
         // fresh connection, including the reconnects the library performs on its own here.
@@ -1609,12 +1669,13 @@ export class CotalEndpoint extends EventEmitter {
      *  auth callout. */
     static RETRY_BACKOFF_CAP_MS = 60_000;
     /** The wait before the next rebuild attempt: {@link retryMs}, doubling per consecutive failure up
-     *  to {@link RETRY_BACKOFF_CAP_MS}. The FIRST retry still waits exactly retryMs, so a transient
-     *  drop recovers as fast as it always did; only a failure that repeats gets paced. */
+     *  to {@link RETRY_BACKOFF_CAP_MS}, then a random half of it taken off. A transient drop still
+     *  recovers within retryMs, and endpoints refused together, as a broker restart or an auth limit
+     *  refuses a host's agents, spread their retries instead of arriving at the callout at once. */
     nextRetryDelayMs() {
         const delay = Math.min(this.retryMs * 2 ** this.retryAttempt, CotalEndpoint.RETRY_BACKOFF_CAP_MS);
         this.retryAttempt++;
-        return delay;
+        return Math.round(delay / 2 + Math.random() * (delay / 2));
     }
     /** Rebuild with backoff until it sticks or we're stopped. Interruptible: a manual
      *  {@link reconnect} kicks the backoff so the next attempt runs immediately instead of
@@ -1632,13 +1693,17 @@ export class CotalEndpoint extends EventEmitter {
                     return; // success — re-armed; the supervisor re-triggers on the next terminal close
                 }
                 catch (e) {
-                    if (!this.stopped)
-                        this.emitRecoverable(e);
+                    const delay = this.nextRetryDelayMs();
+                    // The broker never says why the auth callout refused a bearer, so a user-mode refusal is
+                    // named as one, with where the reason is, and paced like any other failure.
+                    if (!this.stopped && !(e instanceof RepeatedRefusal))
+                        this.emitRecoverable(this.userMode && isAuthRejection(e)
+                            ? new Error(`the broker refused this endpoint's bearer for ${this.owner}.${this.actor} - retrying in ${Math.ceil(delay / 1000)}s; the space's auth service logs why`, { cause: e })
+                            : e);
                     // A warning listener may stop the endpoint. A backoff armed after that holds the process
                     // open for the whole delay, because the stop has already kicked the backoff it saw.
                     if (this.stopped)
                         return;
-                    const delay = this.nextRetryDelayMs();
                     await new Promise((resolve) => {
                         this.backoffResolve = resolve;
                         this.backoffTimer = setTimeout(resolve, delay);
@@ -1700,8 +1765,7 @@ export class CotalEndpoint extends EventEmitter {
             clearInterval(this.heartbeatTimer);
         if (this.sweepTimer)
             clearInterval(this.sweepTimer);
-        if (this.bearerTimer)
-            clearTimeout(this.bearerTimer);
+        this.cancelBearerRenewal?.();
         if (this.credsTimer)
             clearTimeout(this.credsTimer);
         if (this.authExpiryReconnectTimer)
@@ -2138,6 +2202,12 @@ export class CotalEndpoint extends EventEmitter {
         const m = await this.nc.request(reqSubject, JSON.stringify(body), { timeout: timeoutMs, noMux: true, reply });
         return m.json();
     }
+    /** The caller this connection's endpoint requests ride as: the legacy triple, or the issued
+     *  caller with its discovered generation (SPEC 13.15). It names the reply rail this connection's
+     *  credential reads, which is where a manager's turn notice for it lands (SPEC 14.3). */
+    serviceRailCaller() {
+        return this.serviceCaller();
+    }
     /** This endpoint's v0.4 caller triple (§13.2) — the identity its minted ep-rail rows pin. The
      *  owner/actor principal is mode-correct by construction (static: DEV_OWNER + the connection
      *  identity; user mode: the bearer's callout-derived pair), and the lifecycle UID is the
@@ -7469,6 +7539,11 @@ function authOpts(a) {
 /** The bearer's `exp` as epoch ms — what the refresh schedule keys on. A bearer without a numeric
  *  `exp` is structurally unusable for a refreshing endpoint (fail-loud, like the principal decode). */
 function bearerExpiryMs(bearer) {
+    return bearerLifetimeMs(bearer, 0).expiresAtMs;
+}
+/** The bearer's lifetime as epoch ms: `iat` to `exp`, starting at `adoptedAtMs` when it carries no
+ *  `iat`. */
+function bearerLifetimeMs(bearer, adoptedAtMs) {
     const payload = bearer.split(".")[1];
     if (!payload)
         throw new Error("user-mode bearer is not a JWT (no payload segment)");
@@ -7481,7 +7556,17 @@ function bearerExpiryMs(bearer) {
     }
     if (typeof claims.exp !== "number")
         throw new Error("user-mode bearer is missing a numeric exp claim");
-    return claims.exp * 1000;
+    const expiresAtMs = claims.exp * 1000;
+    return { issuedAtMs: Math.min(typeof claims.iat === "number" ? claims.iat * 1000 : adoptedAtMs, expiresAtMs), expiresAtMs };
+}
+/** Past this point a held bearer is overdue: its renewal should already have replaced it. */
+function bearerRenewalDeadlineMs(bearer, adoptedAtMs) {
+    const { issuedAtMs, expiresAtMs } = bearerLifetimeMs(bearer, adoptedAtMs);
+    return renewalDeadlineMs(issuedAtMs, expiresAtMs);
+}
+/** A refusal already surfaced once for this outage. The rebuild loop retries it without a notice,
+ *  so an endpoint that cannot renew shows one status line rather than one per attempt. */
+class RepeatedRefusal extends Error {
 }
 /** Decode the owner+actor PRINCIPAL from a user bearer WITHOUT verifying it — the client trusts its own
  *  bearer only to build its subjects; the broker's minted grant (from the callout, which DOES verify the
@@ -7741,6 +7826,12 @@ function tcpDialable(server, timeoutMs) {
 export function isAuthRejection(e) {
     return e instanceof AuthorizationError || e instanceof UserAuthenticationExpiredError;
 }
+/** The wait a credential source asked for with a numeric `retryAfterSeconds` on its error, in
+ *  milliseconds, or 0. */
+function retryAfterMs(e) {
+    const seconds = e?.retryAfterSeconds;
+    return typeof seconds === "number" && Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : 0;
+}
 /** Whether a NATS server is *running* at `servers`. With NO creds this is a SILENT plaintext
  *  liveness check ({@link tcpInfoProbe}): it reads the server's pre-auth `INFO` greeting and closes
  *  WITHOUT authenticating, so a live broker (open OR auth — INFO precedes auth) returns true while