@cotal-ai/core 0.171.2 → 0.171.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-file.d.ts +3 -2
- package/dist/agent-file.d.ts.map +1 -1
- package/dist/agent-file.js +11 -2
- package/dist/agent-file.js.map +1 -1
- package/dist/endpoint-credentials.d.ts +306 -0
- package/dist/endpoint-credentials.d.ts.map +1 -0
- package/dist/endpoint-credentials.js +585 -0
- package/dist/endpoint-credentials.js.map +1 -0
- package/dist/endpoint.d.ts +20 -159
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +95 -487
- package/dist/endpoint.js.map +1 -1
- package/package.json +1 -1
package/dist/endpoint.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { EndpointHistory } from "./endpoint-history.js";
|
|
2
2
|
export { MULTI_FILTER_BATCH, MULTI_FILTER_READ_CONCURRENCY } from "./endpoint-history.js";
|
|
3
|
-
import { requestSafeReconnect } from "./request-safe-reconnect.js";
|
|
4
3
|
import { nativeBindingAfter, nativeConnectionBinding } from "./native-connection.js";
|
|
5
4
|
import { EventEmitter } from "node:events";
|
|
6
5
|
import { drainAndClose } from "./connection-cleanup.js";
|
|
@@ -9,8 +8,7 @@ import { nanos, PermissionViolationError, NoRespondersError, RequestError, Timeo
|
|
|
9
8
|
import { responderFromLease, responderFromProbe, } from "./liveness.js";
|
|
10
9
|
import { EndpointPresence } from "./endpoint-presence.js";
|
|
11
10
|
export { PresenceWriteStuckError } from "./endpoint-presence.js";
|
|
12
|
-
import { credsClaims,
|
|
13
|
-
import { armRenewal, jitteredBackoffMs } from "./renewal.js";
|
|
11
|
+
import { credsClaims, credsRenewalDelayMs, idFromCreds } from "./identity.js";
|
|
14
12
|
import { requireBrokerFloor } from "./broker-floor.js";
|
|
15
13
|
import { withDeadline } from "./deadline.js";
|
|
16
14
|
import {} from "./secret-store.js";
|
|
@@ -42,6 +40,7 @@ import { EndpointDurableBackstop } from "./endpoint-durable-backstop.js";
|
|
|
42
40
|
import { EndpointSubscriptions, membershipConsumerReleased } from "./endpoint-subscriptions.js";
|
|
43
41
|
import { EndpointChannelDiscovery } from "./endpoint-channel-discovery.js";
|
|
44
42
|
import { EndpointLiveness } from "./endpoint-liveness.js";
|
|
43
|
+
import { EndpointCredentials } from "./endpoint-credentials.js";
|
|
45
44
|
import { EndpointProvisioning } from "./endpoint-provisioning.js";
|
|
46
45
|
import { EndpointBootMemberships } from "./endpoint-boot-memberships.js";
|
|
47
46
|
import { ackWaitOf, durableDelivery, isRecord } from "./endpoint-inbound.js";
|
|
@@ -51,7 +50,7 @@ import { ANYCAST_CANCEL_OP, ANYCAST_NOTICE_KEY, ANYCAST_RECORDS_CREDENTIAL_OUTDA
|
|
|
51
50
|
import { assertValidRole, chatStream, chatSubject, controlServiceSubject, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, membershipInAllow, isConcreteChannel, isChannelFamily, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
|
|
52
51
|
import { isPermissionDenied } from "./permission-denied.js";
|
|
53
52
|
export { isPermissionDenied };
|
|
54
|
-
import { DEFAULT_SERVER, isPublishPermissionDenied, dialerFor,
|
|
53
|
+
import { DEFAULT_SERVER, isPublishPermissionDenied, dialerFor, isAuthRejection, kvWatchConsumer, decodeBearerPrincipal, bearerRenewalDeadlineMs, authOpts, RepeatedRefusal, describeStatusError, } from "./endpoint-dial.js";
|
|
55
54
|
export { DEFAULT_SERVER, isPublishPermissionDenied, wsServers, defaultProbeTimeoutMs, dialerFor, isAuthRejection, isReachable, probeConnect, } from "./endpoint-dial.js";
|
|
56
55
|
import { MAX_TIMER_MS, ANYCAST_SWEEP_MS, MAX_TAKES_IN_FLIGHT, ANYCAST_TAKE_TIMEOUT_MS, DmDurableMissingError, DmBindRefusedError, } from "./endpoint-options.js";
|
|
57
56
|
export { deliveryLimitOf, DEFAULT_SPACE, DmDurableMissingError, DmBindRefusedError, fitHistoryPage, } from "./endpoint-options.js";
|
|
@@ -637,12 +636,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
637
636
|
return;
|
|
638
637
|
this.superviseConnection();
|
|
639
638
|
}
|
|
640
|
-
/** The floor on a bearer renewal's delay. A token whose whole lifetime sits under it is renewed at
|
|
641
|
-
* this cadence (`expiry-renewal` mints 5s bearers). */
|
|
642
|
-
static BEARER_MIN_RENEWAL_MS = 5_000;
|
|
643
|
-
/** A failed bearer renewal retries after about this long, doubling per failure up to the cap. */
|
|
644
|
-
static BEARER_RETRY_MS = 5_000;
|
|
645
|
-
static BEARER_RETRY_CAP_MS = 60_000;
|
|
646
639
|
/**
|
|
647
640
|
* A condition the endpoint is already surviving. Node rethrows `error` when no listener is
|
|
648
641
|
* attached, so emitting retry notices on `error` killed hosts that the endpoint intended to
|
|
@@ -651,461 +644,115 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
651
644
|
emitRecoverable(err) {
|
|
652
645
|
this.emit("warning", err);
|
|
653
646
|
}
|
|
654
|
-
/**
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
*
|
|
658
|
-
*
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
}
|
|
718
|
-
}
|
|
719
|
-
armBearerTimer(cancel) {
|
|
720
|
-
this.cancelBearerRenewal?.();
|
|
721
|
-
this.cancelBearerRenewal = cancel;
|
|
722
|
-
if (this.stopped)
|
|
723
|
-
cancel();
|
|
724
|
-
}
|
|
725
|
-
/** The timed renewal. A new token moves the live connection onto itself at once, through the
|
|
726
|
-
* same controlled reconnect the creds path uses, so the broker never has to expire a connection
|
|
727
|
-
* this endpoint could renew. With no live connection, it ends the rebuild loop's wait instead:
|
|
728
|
-
* that loop refuses to dial an expired token and leaves the retries to this timer. */
|
|
729
|
-
async renewBearerOnTimer() {
|
|
730
|
-
const held = this.currentBearer;
|
|
731
|
-
await this.refreshBearer();
|
|
732
|
-
if (this.stopped || this.currentBearer === held)
|
|
733
|
-
return;
|
|
734
|
-
if (this.nc && !this.nc.isClosed())
|
|
735
|
-
await this.swapConnectionOntoFreshCreds();
|
|
736
|
-
else
|
|
737
|
-
this.kickBackoff();
|
|
647
|
+
/** The resident swap a reload scheduled, until its connection is back or the gate's bound ends. */
|
|
648
|
+
residentSwap;
|
|
649
|
+
/** Serializes every prove-then-adopt transaction (the 75% timer and the explicit reload) so a timer
|
|
650
|
+
* tick and an explicit reload can never interleave their fetch/preflight/commit — the design's
|
|
651
|
+
* single-flight requirement. Runs `fn` after any in-flight transaction settles, whatever its
|
|
652
|
+
* outcome; the internal chain never rejects. */
|
|
653
|
+
credsTxn = Promise.resolve();
|
|
654
|
+
/** The bearer and credential renewal of this endpoint. Every dependency is read when it is used, so
|
|
655
|
+
* a field assigned after construction is seen. */
|
|
656
|
+
credentials = new EndpointCredentials({
|
|
657
|
+
nc: () => this.nc,
|
|
658
|
+
bearerSource: () => this.bearerSource,
|
|
659
|
+
owner: () => this.owner,
|
|
660
|
+
actor: () => this.actor,
|
|
661
|
+
space: () => this.space,
|
|
662
|
+
currentBearer: () => this.currentBearer,
|
|
663
|
+
setCurrentBearer: (value) => { this.currentBearer = value; },
|
|
664
|
+
bearerAdoptedAtMs: () => this.bearerAdoptedAtMs,
|
|
665
|
+
setBearerAdoptedAtMs: (value) => { this.bearerAdoptedAtMs = value; },
|
|
666
|
+
bearerRenewalFailures: () => this.bearerRenewalFailures,
|
|
667
|
+
setBearerRenewalFailures: (value) => { this.bearerRenewalFailures = value; },
|
|
668
|
+
cancelBearerRenewal: () => this.cancelBearerRenewal,
|
|
669
|
+
setCancelBearerRenewal: (value) => { this.cancelBearerRenewal = value; },
|
|
670
|
+
stopped: () => this.stopped,
|
|
671
|
+
presentedExp: () => this.presentedExp,
|
|
672
|
+
setPresentedExp: (value) => { this.presentedExp = value; },
|
|
673
|
+
currentCreds: () => this.currentCreds,
|
|
674
|
+
setCurrentCreds: (value) => { this.currentCreds = value; },
|
|
675
|
+
credsSource: () => this.credsSource,
|
|
676
|
+
connId: () => this.connId,
|
|
677
|
+
overlapOnConnectionRenewal: () => this.overlapOnConnectionRenewal,
|
|
678
|
+
onConnection: () => this.onConnection,
|
|
679
|
+
servers: () => this.servers,
|
|
680
|
+
card: () => this.card,
|
|
681
|
+
token: () => this.token,
|
|
682
|
+
user: () => this.user,
|
|
683
|
+
pass: () => this.pass,
|
|
684
|
+
userMode: () => this.userMode,
|
|
685
|
+
sentinelCreds: () => this.sentinelCreds,
|
|
686
|
+
tls: () => this.tls,
|
|
687
|
+
ownedConnections: () => this.ownedConnections,
|
|
688
|
+
authExpiryReconnectTimer: () => this.authExpiryReconnectTimer,
|
|
689
|
+
setAuthExpiryReconnectTimer: (value) => { this.authExpiryReconnectTimer = value; },
|
|
690
|
+
credsTxnsRunning: () => this.credsTxnsRunning,
|
|
691
|
+
setCredsTxnsRunning: (value) => { this.credsTxnsRunning = value; },
|
|
692
|
+
credsTxn: () => this.credsTxn,
|
|
693
|
+
setCredsTxn: (value) => { this.credsTxn = value; },
|
|
694
|
+
reconnecting: () => this.reconnecting,
|
|
695
|
+
reestablishing: () => this.reestablishing,
|
|
696
|
+
requestsAwaitingReply: () => this.requestsAwaitingReply,
|
|
697
|
+
residentSwap: () => this.residentSwap,
|
|
698
|
+
setResidentSwap: (value) => { this.residentSwap = value; },
|
|
699
|
+
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
700
|
+
kickBackoff: () => this.kickBackoff(),
|
|
701
|
+
armCredsRefresh: (delayMs) => this.armCredsRefresh(delayMs),
|
|
702
|
+
reconnect: () => this.reconnect(),
|
|
703
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
704
|
+
on: (event, listener) => { this.on(event, listener); },
|
|
705
|
+
off: (event, listener) => { this.off(event, listener); },
|
|
706
|
+
});
|
|
707
|
+
/** Forwards to the credentials component's {@link EndpointCredentials.refreshBearer}. */
|
|
708
|
+
refreshBearer(initial = false) {
|
|
709
|
+
return this.credentials.refreshBearer(initial);
|
|
738
710
|
}
|
|
739
|
-
/**
|
|
740
|
-
* on its own included, so none presents an expired token, which the callout could only deny
|
|
741
|
-
* after a full round trip. The bearer counterpart of {@link credsForWire}. */
|
|
711
|
+
/** Forwards to the credentials component's {@link EndpointCredentials.bearerForWire}. */
|
|
742
712
|
bearerForWire() {
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
throw new Error(this.bearerSource
|
|
747
|
-
? "this endpoint's user bearer has expired and renewal through the auth exchange is failing - not presenting the expired token to the broker; retrying with backoff"
|
|
748
|
-
: "this endpoint's user bearer has expired and it holds no bearer source to renew it - re-authenticate and rebuild the endpoint (construct it with a bearer FUNCTION for standing renewal)");
|
|
749
|
-
}
|
|
750
|
-
/** How soon a FAILED creds refresh retries. Successful refreshes schedule by lifetime fraction
|
|
751
|
-
* (75% of iat→exp), not a fixed margin — standing creds span hours to days, bearers minutes. */
|
|
752
|
-
static CREDS_RETRY_MS = 60_000;
|
|
753
|
-
/** THE ONE PLACE a credential is cleared for presentation to a broker.
|
|
754
|
-
*
|
|
755
|
-
* The property is unconditional — this endpoint never presents a credential it has already
|
|
756
|
-
* decoded as expired — so it is a property of the SUPPLY, not of any one dial site. It used to
|
|
757
|
-
* live inside {@link bindConnection}, which only `start()` and `doRebuild` reach; the
|
|
758
|
-
* authenticator nats.js re-evaluates on ITS OWN reconnects read the cache directly and so
|
|
759
|
-
* presented whatever was last fetched, expired included. Two such reconnects exist and neither
|
|
760
|
-
* passes through bindConnection: the one the broker forces at JWT `exp`, and an ALREADY RUNNING
|
|
761
|
-
* dial loop from an earlier drop that crosses `exp` while it retries (the pre-expiry
|
|
762
|
-
* reconnect fence flips a policy flag nats-core only reads when it observes a NEW drop, so it
|
|
763
|
-
* cannot stop a loop already in flight).
|
|
764
|
-
*
|
|
765
|
-
* Putting the refusal here instead means a future caller cannot miss it: the only way to reach a
|
|
766
|
-
* dial is through {@link credsForWire}, and the one presentation that does not read the cache
|
|
767
|
-
* (the adoption preflight, which presents a fresh CANDIDATE) calls this same function on it.
|
|
768
|
-
*
|
|
769
|
-
* An unbounded credential (no numeric `exp`) is presentable: bounded lifetimes are the renewal
|
|
770
|
-
* seam's concern, and a cred with no expiry has none to be past. */
|
|
771
|
-
static presentableCreds(creds, opts) {
|
|
772
|
-
const { exp } = credsClaims(creds); // throws on a structurally-unusable file (fail-loud)
|
|
773
|
-
if (typeof exp === "number" && exp * 1000 <= Date.now())
|
|
774
|
-
throw new Error(opts.renewable
|
|
775
|
-
? "this endpoint's creds have expired and renewal is failing - not presenting the expired credential to the broker; retrying with backoff"
|
|
776
|
-
: "this endpoint's creds have expired and it holds no creds source to renew them - replace the credential and rebuild the endpoint (pass a creds FUNCTION for standing renewal)");
|
|
777
|
-
return creds;
|
|
778
|
-
}
|
|
779
|
-
/** The connect attempt's credential getters: what {@link credsForWire} and {@link bearerForWire}
|
|
780
|
-
* return, with its expiry recorded as {@link presentedExp}. */
|
|
713
|
+
return this.credentials.bearerForWire();
|
|
714
|
+
}
|
|
715
|
+
/** Forwards to the credentials component's {@link EndpointCredentials.presentCreds}. */
|
|
781
716
|
presentCreds() {
|
|
782
|
-
|
|
783
|
-
this.presentedExp = credsClaims(creds).exp;
|
|
784
|
-
return creds;
|
|
717
|
+
return this.credentials.presentCreds();
|
|
785
718
|
}
|
|
719
|
+
/** Forwards to the credentials component's {@link EndpointCredentials.presentBearer}. */
|
|
786
720
|
presentBearer() {
|
|
787
|
-
|
|
788
|
-
this.presentedExp = bearerExpiryMs(bearer) / 1000;
|
|
789
|
-
return bearer;
|
|
790
|
-
}
|
|
791
|
-
/** The cached credential, checked. Handed to nats.js as the authenticator's source on EVERY auth
|
|
792
|
-
* mode (renewed or static), so each (re)connect attempt — ours or the library's — re-reads a
|
|
793
|
-
* CHECKED value. A refusal throws out of the authenticator, which nats-core turns into a closed
|
|
794
|
-
* connection rather than a CONNECT carrying dead material; the endpoint's own supervisor then
|
|
795
|
-
* rebuilds on capped backoff, and {@link bindConnection} re-fetches from the source on each of
|
|
796
|
-
* those attempts, so a renewal that starts working recovers the endpoint without presenting
|
|
797
|
-
* anything expired in the meantime. Deliberately side-effect free: kicking the renewal timer
|
|
798
|
-
* from here would retry the source once per dial attempt, which is the flat load on a dead
|
|
799
|
-
* broker that {@link RETRY_BACKOFF_CAP_MS} exists to prevent. */
|
|
800
|
-
credsForWire() {
|
|
801
|
-
if (!this.currentCreds)
|
|
802
|
-
throw new Error(this.credsSource
|
|
803
|
-
? "this endpoint has no credential to present yet (the creds source has not returned one) - not dialing without auth material"
|
|
804
|
-
: "this endpoint was constructed with an empty creds string - not dialing without auth material (an empty credential is not anonymous access)");
|
|
805
|
-
return CotalEndpoint.presentableCreds(this.currentCreds, { renewable: Boolean(this.credsSource) });
|
|
721
|
+
return this.credentials.presentBearer();
|
|
806
722
|
}
|
|
807
723
|
/** The credential this endpoint presents on its next connect, checked as {@link credsForWire}
|
|
808
724
|
* checks it. A second connection under the same identity reads it on every attempt, so it
|
|
809
725
|
* presents each credential this endpoint adopts and never an expired one. */
|
|
810
726
|
wireCreds() {
|
|
811
|
-
return this.
|
|
727
|
+
return this.credentials.wireCreds();
|
|
812
728
|
}
|
|
813
|
-
/**
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
* A source that hands back the SAME generation past its own renewal point has not re-signed yet:
|
|
817
|
-
* that is a missed remint, not a candidate. Adopting it re-arms from a non-positive delay, which
|
|
818
|
-
* {@link armCredsRefresh} floors to 1s, which fetches again, which is the same generation — a 1s
|
|
819
|
-
* read loop against a store that is already having a bad day, for the JWT's remaining 25% of life,
|
|
820
|
-
* with the 60s backoff that exists for exactly this bypassed because the fetch did not FAIL.
|
|
821
|
-
*
|
|
822
|
-
* The membership feed's rw-cred renewal already refuses it this way (`membership-feed.ts`,
|
|
823
|
-
* `adoptRwCreds`: "the rw source still holds the previous generation past its renewal point");
|
|
824
|
-
* this is the same rule on the endpoint's `delivery.creds` seam, so the two renewal paths answer
|
|
825
|
-
* a dead source identically.
|
|
826
|
-
*
|
|
827
|
-
* An ALREADY-EXPIRED generation is refused whatever it is: the delay is non-positive for it too,
|
|
828
|
-
* but it is dead rather than merely due, so it is named separately and refused even when the
|
|
829
|
-
* source keeps returning a different one. */
|
|
830
|
-
static assertRenewableGeneration(candidate, current, delayMs) {
|
|
831
|
-
const { exp } = credsClaims(candidate);
|
|
832
|
-
if (typeof exp === "number" && exp * 1000 <= Date.now())
|
|
833
|
-
throw new Error("the creds source returned an already-expired credential (its `exp` is in the past) - nothing adopted; the renewal owner has not re-signed it, or the store is serving a stale generation");
|
|
834
|
-
// Compared by GENERATION, not by envelope. `EndpointOptions.creds` takes opaque file content and
|
|
835
|
-
// promises no canonical whitespace: `jwtFromCreds` pads with `\s*` and trims, so a store, editor or
|
|
836
|
-
// filesystem round trip that adds a newline re-serves the SAME credential in bytes `===` calls
|
|
837
|
-
// different - and the refusal would be skipped for the one case it exists to catch. The envelope
|
|
838
|
-
// also carries the nkey seed, which is why `credsFingerprint` hashes the JWT instead.
|
|
839
|
-
if (current !== undefined && credsFingerprint(candidate) === credsFingerprint(current) && delayMs <= 0)
|
|
840
|
-
throw new Error("the creds source still holds the previous generation past its renewal point (the renewal owner has not re-signed it) - nothing adopted");
|
|
841
|
-
}
|
|
842
|
-
/** The disposable-preflight connect bound for the EXPLICIT reload proof (D5 class-2 adoption). A
|
|
843
|
-
* rogue or unreachable candidate must resolve well UNDER the manager's delivery-admin request
|
|
844
|
-
* bound, so this stays a few seconds and never blocks the responder. */
|
|
845
|
-
static PREFLIGHT_MS = 4_000;
|
|
846
|
-
/** How long the resident wire swap is deferred past the reply on the EXPLICIT reload path. The
|
|
847
|
-
* delivery-admin responder rides THIS connection, so `nc.reconnect()` must not run until the
|
|
848
|
-
* reply has flushed on the still-live old cred; the old cred stays valid until its exp, so a short
|
|
849
|
-
* deferral is safe. */
|
|
850
|
-
static RESIDENT_SWAP_DEFER_MS = 250;
|
|
851
|
-
/** How long a delivery-admin request read after a reload waits for the resident swap's reconnect.
|
|
852
|
-
* nats.js drops a publish made while the connection is down when it redials, so a reply sent
|
|
853
|
-
* during the swap is lost and its caller waits out its whole request timeout. Past this bound the
|
|
854
|
-
* request is answered anyway, as it was before the gate. */
|
|
855
|
-
static RESIDENT_SWAP_GATE_MS = 5_000;
|
|
856
|
-
/** The resident swap a reload scheduled, until its connection is back or the gate's bound ends. */
|
|
857
|
-
residentSwap;
|
|
858
|
-
/** The daemon-side deadline for the WHOLE prove-then-adopt transaction (source fetch + preflight +
|
|
859
|
-
* commit), kept strictly BELOW the manager's delivery-admin request bound so a slow or hung hosted
|
|
860
|
-
* store returns a structured component failure rather than an ambiguous client timeout, and a late
|
|
861
|
-
* fetch can never commit after the caller gave up. */
|
|
862
|
-
static RELOAD_DEADLINE_MS = 12_000;
|
|
863
|
-
/** Fetch fresh creds from the source, pin their identity to ours, cache them, and arm the next
|
|
864
|
-
* refresh at 75% of the new JWT's lifetime. THROWS on fetch/pin failure — the callers decide the
|
|
865
|
-
* failure posture (loud-and-retry for the timer, a structured error reply for an explicit
|
|
866
|
-
* {@link reloadCreds}). This is the PASSIVE-BACKSTOP fetch; the explicit auditable path
|
|
867
|
-
* ({@link reloadCreds}) does its own fetch so it can preflight the candidate on a disposable
|
|
868
|
-
* connection BEFORE mutating this live cache. */
|
|
869
|
-
async fetchFreshCreds() {
|
|
870
|
-
const creds = await this.credsSource();
|
|
871
|
-
const id = idFromCreds(creds);
|
|
872
|
-
if (id !== this.connId)
|
|
873
|
-
throw new Error(`creds source returned identity ${id}, expected ${this.connId} - renewal may not swap the connection's nkey`);
|
|
874
|
-
// Before the commit, so a refusal leaves the cache holding the last generation that was renewable
|
|
875
|
-
// from (and, on the initial fetch, fails loud rather than dialling with dead material).
|
|
876
|
-
const delay = credsRenewalDelayMs(creds);
|
|
877
|
-
CotalEndpoint.assertRenewableGeneration(creds, this.currentCreds, delay);
|
|
878
|
-
this.currentCreds = creds;
|
|
879
|
-
this.armCredsRefresh(delay);
|
|
880
|
-
const { iat, exp } = credsClaims(creds);
|
|
881
|
-
return { iat, exp };
|
|
882
|
-
}
|
|
883
|
-
/** Swap the live connection onto the freshest cached cred or bearer with a controlled
|
|
884
|
-
* `nc.reconnect()` (nats.js re-evaluates the authenticator per attempt). Swapping now, instead of
|
|
885
|
-
* waiting for the broker to close the connection at `exp`, means the wire never carries a
|
|
886
|
-
* near-dead JWT and the operator never sees a spurious "authentication expired". After the
|
|
887
|
-
* reconnect succeeds, re-arm the expiry fence for the credential now authenticated on the wire. */
|
|
888
|
-
async swapConnectionOntoFreshCreds() {
|
|
889
|
-
const nc = this.nc;
|
|
890
|
-
if (!(nc && !this.stopped))
|
|
891
|
-
return;
|
|
892
|
-
const overlap = this.overlapOnConnectionRenewal && this.onConnection
|
|
893
|
-
? await this.openRenewalOverlap().catch(() => undefined)
|
|
894
|
-
: undefined;
|
|
895
|
-
try {
|
|
896
|
-
// reconnect() resolves when the disconnect is requested, before subscriptions return.
|
|
897
|
-
// A flush started now is sent after those subscriptions, so it resolves only once they
|
|
898
|
-
// are on the broker. The overlap stays up until then.
|
|
899
|
-
await nc.reconnect().catch(() => { });
|
|
900
|
-
if (overlap && !nc.isClosed())
|
|
901
|
-
await withDeadline(nc.flush(), 10_000, () => new Error("resident renewal flush did not finish")).catch(() => { });
|
|
902
|
-
if (this.nc === nc && !this.stopped)
|
|
903
|
-
this.armAuthExpiryReconnectFence(nc);
|
|
904
|
-
}
|
|
905
|
-
finally {
|
|
906
|
-
if (overlap && !overlap.isClosed())
|
|
907
|
-
await this.closeWithoutLibraryReconnect(overlap).catch(() => { });
|
|
908
|
-
}
|
|
729
|
+
/** Forwards to the credentials component's {@link EndpointCredentials.disableLibraryReconnect}. */
|
|
730
|
+
disableLibraryReconnect(nc) {
|
|
731
|
+
return this.credentials.disableLibraryReconnect(nc);
|
|
909
732
|
}
|
|
910
|
-
/**
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
async openRenewalOverlap() {
|
|
914
|
-
const overlap = await dialerFor(this.servers)({
|
|
915
|
-
servers: this.servers,
|
|
916
|
-
noAsyncTraces: true,
|
|
917
|
-
name: `cotal:${this.card.name}:renewal`,
|
|
918
|
-
inboxPrefix: `_INBOX_renew_${randomUUID()}`,
|
|
919
|
-
maxReconnectAttempts: 0,
|
|
920
|
-
...authOpts({
|
|
921
|
-
token: this.token, user: this.user, pass: this.pass,
|
|
922
|
-
creds: this.currentCreds !== undefined || this.credsSource ? () => this.presentCreds() : undefined,
|
|
923
|
-
bearer: this.userMode ? () => this.presentBearer() : undefined,
|
|
924
|
-
sentinelCreds: this.sentinelCreds, tls: this.tls,
|
|
925
|
-
}),
|
|
926
|
-
});
|
|
927
|
-
this.ownedConnections.add(overlap);
|
|
928
|
-
try {
|
|
929
|
-
this.onConnection?.(overlap);
|
|
930
|
-
await withDeadline(overlap.flush(), 5_000, () => new Error("renewal overlap flush did not finish"));
|
|
931
|
-
return overlap;
|
|
932
|
-
}
|
|
933
|
-
catch (error) {
|
|
934
|
-
await this.closeWithoutLibraryReconnect(overlap).catch(() => { });
|
|
935
|
-
throw error;
|
|
936
|
-
}
|
|
733
|
+
/** Forwards to the credentials component's {@link EndpointCredentials.closeWithoutLibraryReconnect}. */
|
|
734
|
+
closeWithoutLibraryReconnect(nc) {
|
|
735
|
+
return this.credentials.closeWithoutLibraryReconnect(nc);
|
|
937
736
|
}
|
|
938
|
-
/**
|
|
939
|
-
* same last-resort surface used by {@link closeFailedBind}; there is no public API for changing the
|
|
940
|
-
* reconnect policy of an existing connection. Disabling it does not disable Cotal self-heal: the
|
|
941
|
-
* `nc.closed()` supervisor below rebuilds the endpoint with freshly checked auth material. */
|
|
942
|
-
disableLibraryReconnect(nc) {
|
|
943
|
-
const protocol = nc.protocol;
|
|
944
|
-
if (protocol?.options)
|
|
945
|
-
protocol.options.reconnect = false;
|
|
946
|
-
}
|
|
947
|
-
/** Close a connection whose library reconnect has already been disabled. `drain()` flushes with a
|
|
948
|
-
* PING and waits for the matching PONG; nats-core only rejects that waiter inside reconnect
|
|
949
|
-
* `prepare()`, so a half-open socket with reconnect=false leaves drain pending until the 2-minute
|
|
950
|
-
* ping interval times out. `close()` tears the transport down without that round-trip, which is
|
|
951
|
-
* the same public path {@link closeFailedBind} already uses when there is no graceful delivery
|
|
952
|
-
* contract left. */
|
|
953
|
-
async closeWithoutLibraryReconnect(nc) {
|
|
954
|
-
if (!nc)
|
|
955
|
-
return;
|
|
956
|
-
this.disableLibraryReconnect(nc);
|
|
957
|
-
await nc.close();
|
|
958
|
-
if (!nc.isClosed())
|
|
959
|
-
throw new Error("endpoint connection closure is unproved; consumer custody must be retained");
|
|
960
|
-
this.ownedConnections.delete(nc);
|
|
961
|
-
}
|
|
962
|
-
/** Disable nats-core reconnect shortly before the JWT authenticated on this wire expires. The small
|
|
963
|
-
* lead makes the policy change precede the broker's expiry close even when both timers wake in the
|
|
964
|
-
* same event-loop turn. A credential adoption does not move this fence until the resident reconnect
|
|
965
|
-
* is requested, so the old wire remains protected during the prove-then-adopt window. */
|
|
737
|
+
/** Forwards to the credentials component's {@link EndpointCredentials.armAuthExpiryReconnectFence}. */
|
|
966
738
|
armAuthExpiryReconnectFence(nc) {
|
|
967
|
-
|
|
968
|
-
this.authExpiryReconnectTimer = undefined;
|
|
969
|
-
const credsExp = this.currentCreds && credsClaims(this.currentCreds).exp;
|
|
970
|
-
const expiryMs = this.userMode && this.currentBearer
|
|
971
|
-
? bearerExpiryMs(this.currentBearer)
|
|
972
|
-
: typeof credsExp === "number"
|
|
973
|
-
? credsExp * 1000
|
|
974
|
-
: undefined;
|
|
975
|
-
if (expiryMs === undefined || this.stopped)
|
|
976
|
-
return;
|
|
977
|
-
// Node fires a delay above 2^31-1 ms (about 24.8 days) after 1 ms, which would disable reconnect
|
|
978
|
-
// at once. A later expiry waits in steps of that bound, and the fence is armed again from there.
|
|
979
|
-
const delay = Math.max(0, expiryMs - Date.now() - 500);
|
|
980
|
-
this.authExpiryReconnectTimer = delay > MAX_TIMER_MS
|
|
981
|
-
? setTimeout(() => { if (this.nc === nc)
|
|
982
|
-
this.armAuthExpiryReconnectFence(nc); }, MAX_TIMER_MS)
|
|
983
|
-
: setTimeout(() => { if (this.nc === nc)
|
|
984
|
-
this.disableLibraryReconnect(nc); }, delay);
|
|
985
|
-
this.authExpiryReconnectTimer.unref?.();
|
|
986
|
-
}
|
|
987
|
-
/** The connectAndBind PRE-CONNECT fetch: pull the freshest source cred and pin it into
|
|
988
|
-
* {@link currentCreds} so the connect() that immediately follows presents it — that connect IS the
|
|
989
|
-
* proof, so NO preflight and NO live swap here. THROWS when `initial` (no cred to start from), else
|
|
990
|
-
* emits and retries. The LIVE-SWAP paths — the 75% timer {@link renewCredsOnTimer} and the explicit
|
|
991
|
-
* {@link reloadCreds} — preflight the candidate on a disposable connection instead, so they never
|
|
992
|
-
* reconnect the live connection onto an unproven cred. */
|
|
993
|
-
async refreshCreds(initial = false) {
|
|
994
|
-
try {
|
|
995
|
-
await this.fetchFreshCreds();
|
|
996
|
-
}
|
|
997
|
-
catch (e) {
|
|
998
|
-
if (initial)
|
|
999
|
-
throw e;
|
|
1000
|
-
this.emitRecoverable(new Error(`creds refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current JWT's expiry if renewal keeps failing`));
|
|
1001
|
-
this.armCredsRefresh(CotalEndpoint.CREDS_RETRY_MS);
|
|
1002
|
-
}
|
|
739
|
+
return this.credentials.armAuthExpiryReconnectFence(nc);
|
|
1003
740
|
}
|
|
1004
|
-
/**
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
* outcome; the internal chain never rejects. */
|
|
1008
|
-
credsTxn = Promise.resolve();
|
|
1009
|
-
runCredsTxn(fn) {
|
|
1010
|
-
const counted = async () => {
|
|
1011
|
-
this.credsTxnsRunning++;
|
|
1012
|
-
try {
|
|
1013
|
-
return await fn();
|
|
1014
|
-
}
|
|
1015
|
-
finally {
|
|
1016
|
-
this.credsTxnsRunning--;
|
|
1017
|
-
}
|
|
1018
|
-
};
|
|
1019
|
-
const next = this.credsTxn.then(counted, counted);
|
|
1020
|
-
this.credsTxn = next.then(() => undefined, () => undefined);
|
|
1021
|
-
return next;
|
|
741
|
+
/** Forwards to the credentials component's {@link EndpointCredentials.refreshCreds}. */
|
|
742
|
+
refreshCreds(initial = false) {
|
|
743
|
+
return this.credentials.refreshCreds(initial);
|
|
1022
744
|
}
|
|
1023
|
-
/**
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
* commit {@link currentCreds} → arm the next 75% timer. Because currentCreds is written ONLY after
|
|
1027
|
-
* the preflight proves broker acceptance, the resident authenticator getter can never present an
|
|
1028
|
-
* unproven cred (not even on an incidental reconnect), and a rejected candidate leaves the resident
|
|
1029
|
-
* connection untouched on BOTH the timer and explicit paths. THROWS (structured) on any failure. */
|
|
1030
|
-
async adoptFreshCreds(opts = {}) {
|
|
1031
|
-
// ABSOLUTE deadline, captured at the CALLER's entry (reloadCreds / renewCredsOnTimer) BEFORE the
|
|
1032
|
-
// single-flight enqueue, so the queue wait behind an in-flight transaction counts against it. A
|
|
1033
|
-
// reload queued behind a hung-store timer tick can never receive a FRESH budget and commit after
|
|
1034
|
-
// the renewal owner's request bound already elapsed.
|
|
1035
|
-
const deadline = opts.deadline ?? Date.now() + CotalEndpoint.RELOAD_DEADLINE_MS;
|
|
1036
|
-
// Fence BEFORE any source I/O: if the queue wait already burned the budget, fail structured now and
|
|
1037
|
-
// never touch the store, the preflight, or currentCreds.
|
|
1038
|
-
if (Date.now() > deadline)
|
|
1039
|
-
throw new Error("reloadCreds: the request deadline elapsed while queued behind another credential transaction; nothing adopted");
|
|
1040
|
-
const candidate = await withDeadline(this.credsSource(), deadline - Date.now(), () => new Error("reloadCreds: the creds source did not return before the daemon deadline; nothing adopted"));
|
|
1041
|
-
const id = idFromCreds(candidate);
|
|
1042
|
-
if (id !== this.connId)
|
|
1043
|
-
throw new Error(`creds source returned identity ${id}, expected ${this.connId} - renewal may not swap the connection's nkey`);
|
|
1044
|
-
// Non-material message ON PURPOSE: this text flows to the manager's persisted
|
|
1045
|
-
// `RenewalRecord.adoption.error`, so neither the observed nor the expected digest may appear.
|
|
1046
|
-
if (opts.expected !== undefined && credsFingerprint(candidate) !== opts.expected)
|
|
1047
|
-
throw new Error("reloadCreds: re-read credential generation did not match the expected re-signed generation (a different store, or a torn/stale read); nothing adopted");
|
|
1048
|
-
// PREFLIGHT = the proof. A disposable connection presenting exactly the candidate BEFORE the live
|
|
1049
|
-
// cache is touched; a refused cred throws here, leaving the resident connection untouched.
|
|
1050
|
-
// The candidate goes through the SAME checkpoint the resident getter uses: this is the one
|
|
1051
|
-
// presentation that does not read the cache, so routing it here is what makes the refusal a
|
|
1052
|
-
// property of every path rather than of the cached one. An already-dead re-signed generation is
|
|
1053
|
-
// refused locally instead of spending a round trip to be told so, and — because this throws
|
|
1054
|
-
// BEFORE the commit below — it can never become the resident connection's next-presented cred.
|
|
1055
|
-
const proven = CotalEndpoint.presentableCreds(candidate, { renewable: true });
|
|
1056
|
-
const probe = await probeConnect(this.servers, { creds: proven, tls: this.tls, timeoutMs: Math.max(500, Math.min(CotalEndpoint.PREFLIGHT_MS, deadline - Date.now())) });
|
|
1057
|
-
if (!probe.ok)
|
|
1058
|
-
throw new Error(`reloadCreds: the broker did not accept the re-signed credential (${probe.reason}); nothing adopted`);
|
|
1059
|
-
if (Date.now() > deadline)
|
|
1060
|
-
throw new Error("reloadCreds: the proof exceeded the daemon deadline; nothing adopted"); // fence a late commit
|
|
1061
|
-
// Validate the bounded-window renewal delay BEFORE the commit: credsRenewalDelayMs throws on a cred
|
|
1062
|
-
// lacking a numeric `exp`, and that throw must not leave currentCreds flipped to a candidate the
|
|
1063
|
-
// authenticator would present on the next reconnect (a post-preflight validation failure is a no-op).
|
|
1064
|
-
const delay = credsRenewalDelayMs(candidate);
|
|
1065
|
-
CotalEndpoint.assertRenewableGeneration(candidate, this.currentCreds, delay);
|
|
1066
|
-
this.currentCreds = candidate;
|
|
1067
|
-
this.armCredsRefresh(delay);
|
|
1068
|
-
this.emit("creds-adopted", credsClaims(candidate));
|
|
1069
|
-
return credsClaims(candidate);
|
|
1070
|
-
}
|
|
1071
|
-
/** The 75%-of-lifetime renewal timer tick: prove + adopt + swap the LIVE connection. Preflights (it
|
|
1072
|
-
* reconnects a live connection, so the candidate must be broker-proven first) and is serialized
|
|
1073
|
-
* with the explicit reload. Never throws — a failed renewal logs and retries while the old cred
|
|
1074
|
-
* stays live until its expiry. */
|
|
1075
|
-
async renewCredsOnTimer() {
|
|
1076
|
-
// Deadline captured HERE, before the enqueue, for the same reason as the explicit reload.
|
|
1077
|
-
const deadline = Date.now() + CotalEndpoint.RELOAD_DEADLINE_MS;
|
|
1078
|
-
try {
|
|
1079
|
-
await this.runCredsTxn(() => this.adoptFreshCreds({ deadline }));
|
|
1080
|
-
await this.swapConnectionOntoFreshCreds();
|
|
1081
|
-
}
|
|
1082
|
-
catch (e) {
|
|
1083
|
-
this.emitRecoverable(new Error(`creds refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current JWT's expiry if renewal keeps failing`));
|
|
1084
|
-
this.armCredsRefresh(CotalEndpoint.CREDS_RETRY_MS);
|
|
1085
|
-
}
|
|
745
|
+
/** Forwards to the credentials component's {@link EndpointCredentials.renewCredsOnTimer}. */
|
|
746
|
+
renewCredsOnTimer() {
|
|
747
|
+
return this.credentials.renewCredsOnTimer();
|
|
1086
748
|
}
|
|
1087
749
|
/** Fetch, broker-prove and adopt the source credential, then refresh the same connection
|
|
1088
750
|
* once outgoing mux and named-reply requests are quiet. A busy timeout retains the connection
|
|
1089
751
|
* and adopted credential. Call again to retry. This does not tear down endpoint bindings.
|
|
1090
752
|
* A dropped connection that is being re-established carries no request, so it redials now
|
|
1091
753
|
* instead of after its backoff; the rebuild re-fetches a stale credential from the source. */
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
throw new Error("credential refresh requires a creds source");
|
|
1095
|
-
const nc = this.nc;
|
|
1096
|
-
if (!nc && !this.stopped && (this.reconnecting || this.reestablishing))
|
|
1097
|
-
return await this.reconnect();
|
|
1098
|
-
if (!nc || this.stopped)
|
|
1099
|
-
throw new Error("credential refresh requires a running endpoint");
|
|
1100
|
-
const deadline = Date.now() + CotalEndpoint.RELOAD_DEADLINE_MS;
|
|
1101
|
-
await this.runCredsTxn(async () => {
|
|
1102
|
-
await this.adoptFreshCreds({ deadline });
|
|
1103
|
-
if (this.nc !== nc || this.stopped)
|
|
1104
|
-
throw new Error("endpoint connection changed during credential refresh");
|
|
1105
|
-
await requestSafeReconnect(nc, () => this.requestsAwaitingReply.get(nc)?.size ?? 0);
|
|
1106
|
-
if (this.nc === nc && !this.stopped)
|
|
1107
|
-
this.armAuthExpiryReconnectFence(nc);
|
|
1108
|
-
});
|
|
754
|
+
refreshCredsRequestSafe() {
|
|
755
|
+
return this.credentials.refreshCredsRequestSafe();
|
|
1109
756
|
}
|
|
1110
757
|
/** EXPLICIT credential reload — the auditable adoption step of D5 class-2 standing renewal (served
|
|
1111
758
|
* to the renewal owner via the delivery-admin rail). The PROOF-OF-RECORD is a DISPOSABLE PREFLIGHT
|
|
@@ -1122,51 +769,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1122
769
|
* the resident wire swap is best-effort and self-healing (the 75% timer + the broker's expiry-close),
|
|
1123
770
|
* NOT witnessed, so the caller must claim broker acceptance, not verified resident reauth. NEVER a
|
|
1124
771
|
* fingerprint. */
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
// 75% timer tick is bounded by the SAME window (queue wait included) and cannot commit/swap after
|
|
1130
|
-
// the renewal owner's request bound has already elapsed and it recorded "no responder".
|
|
1131
|
-
const deadline = Date.now() + CotalEndpoint.RELOAD_DEADLINE_MS;
|
|
1132
|
-
// The prove-then-adopt transaction (fetch + preflight + commit) runs under the single-flight so it
|
|
1133
|
-
// cannot interleave with the 75% timer; the wire swap is scheduled by {@link handleDeliveryAdmin}
|
|
1134
|
-
// after the aggregate reply (the responder rides this connection).
|
|
1135
|
-
const { iat, exp } = await this.runCredsTxn(() => this.adoptFreshCreds({ expected, deadline }));
|
|
1136
|
-
return { identity: this.connId, iat, exp };
|
|
1137
|
-
}
|
|
1138
|
-
/** Arm the resident wire swap for the delivery-admin `reloadCreds` path. Called by
|
|
1139
|
-
* {@link handleDeliveryAdmin} ONLY after BOTH component proofs have settled and immediately before
|
|
1140
|
-
* the reply is returned+responded — never from inside {@link reloadCreds} while the aggregate is
|
|
1141
|
-
* still open, or a slow co-component proof would let `nc.reconnect()` reconnect the admin rail
|
|
1142
|
-
* before the reply flushes and silently strand it. The short deferral lets the reply flush on the
|
|
1143
|
-
* still-live old cred first. Best-effort — the proof was the preflight; a transient swap failure is
|
|
1144
|
-
* retried by the 75% timer and the broker's own expiry-close, both presenting the adopted candidate.
|
|
1145
|
-
* Until the swapped connection is back, {@link residentSwap} holds the delivery-admin requests that
|
|
1146
|
-
* arrive after this reply. */
|
|
772
|
+
reloadCreds(expected) {
|
|
773
|
+
return this.credentials.reloadCreds(expected);
|
|
774
|
+
}
|
|
775
|
+
/** Forwards to the credentials component's {@link EndpointCredentials.scheduleResidentSwap}. */
|
|
1147
776
|
scheduleResidentSwap() {
|
|
1148
|
-
|
|
1149
|
-
return;
|
|
1150
|
-
const swap = new Promise((resolve) => { setTimeout(resolve, CotalEndpoint.RESIDENT_SWAP_DEFER_MS).unref?.(); })
|
|
1151
|
-
.then(async () => {
|
|
1152
|
-
const back = this.transportBack(CotalEndpoint.RESIDENT_SWAP_GATE_MS);
|
|
1153
|
-
await this.swapConnectionOntoFreshCreds();
|
|
1154
|
-
await back;
|
|
1155
|
-
})
|
|
1156
|
-
.finally(() => { if (this.residentSwap === swap)
|
|
1157
|
-
this.residentSwap = undefined; });
|
|
1158
|
-
this.residentSwap = swap;
|
|
1159
|
-
}
|
|
1160
|
-
/** Resolve when this endpoint reports its transport connected again, or after `ms`. */
|
|
1161
|
-
transportBack(ms) {
|
|
1162
|
-
return new Promise((resolve) => {
|
|
1163
|
-
const done = () => { clearTimeout(timer); this.off("transport", seen); resolve(); };
|
|
1164
|
-
const seen = (state) => { if (state.connected)
|
|
1165
|
-
done(); };
|
|
1166
|
-
const timer = setTimeout(done, ms);
|
|
1167
|
-
timer.unref?.();
|
|
1168
|
-
this.on("transport", seen);
|
|
1169
|
-
});
|
|
777
|
+
return this.credentials.scheduleResidentSwap();
|
|
1170
778
|
}
|
|
1171
779
|
armCredsRefresh(delayMs) {
|
|
1172
780
|
if (this.stopped)
|