@cotal-ai/core 0.171.3 → 0.171.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/endpoint-connection.d.ts +363 -0
- package/dist/endpoint-connection.d.ts.map +1 -0
- package/dist/endpoint-connection.js +741 -0
- package/dist/endpoint-connection.js.map +1 -0
- package/dist/endpoint-reconnect.d.ts +129 -0
- package/dist/endpoint-reconnect.d.ts.map +1 -0
- package/dist/endpoint-reconnect.js +166 -0
- package/dist/endpoint-reconnect.js.map +1 -0
- package/dist/endpoint.d.ts +17 -55
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +231 -680
- package/dist/endpoint.js.map +1 -1
- package/package.json +1 -1
package/dist/endpoint.js
CHANGED
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
import { EndpointHistory } from "./endpoint-history.js";
|
|
2
2
|
export { MULTI_FILTER_BATCH, MULTI_FILTER_READ_CONCURRENCY } from "./endpoint-history.js";
|
|
3
|
-
import { nativeBindingAfter
|
|
3
|
+
import { nativeBindingAfter } from "./native-connection.js";
|
|
4
4
|
import { EventEmitter } from "node:events";
|
|
5
|
-
import { drainAndClose } from "./connection-cleanup.js";
|
|
6
5
|
import { randomUUID } from "node:crypto";
|
|
7
6
|
import { nanos, PermissionViolationError, NoRespondersError, RequestError, TimeoutError, } from "@nats-io/transport-node";
|
|
8
7
|
import { responderFromLease, responderFromProbe, } from "./liveness.js";
|
|
9
8
|
import { EndpointPresence } from "./endpoint-presence.js";
|
|
10
9
|
export { PresenceWriteStuckError } from "./endpoint-presence.js";
|
|
11
|
-
import {
|
|
12
|
-
import { requireBrokerFloor } from "./broker-floor.js";
|
|
10
|
+
import { idFromCreds } from "./identity.js";
|
|
13
11
|
import { withDeadline } from "./deadline.js";
|
|
14
12
|
import {} from "./secret-store.js";
|
|
15
13
|
import { BIND_SPLIT_REISSUES, resolveService } from "./endpoint-invoke.js";
|
|
@@ -20,14 +18,13 @@ import { servePrincipalRoute, describeContracts } from "./endpoint-serve.js";
|
|
|
20
18
|
import { epPrincipalCall } from "./endpoint-call.js";
|
|
21
19
|
import { AGENT_CLUSTER, AGENT_CONTRACTS, AGENT_ENDPOINT } from "./agent-endpoint.js";
|
|
22
20
|
import { processRuntime } from "./process-runtime.js";
|
|
23
|
-
import {
|
|
21
|
+
import { issuedPermitsSubject } from "./issued-authority.js";
|
|
24
22
|
import { listRegistryEndpoints } from "./registry-client.js";
|
|
25
23
|
import { isConsumerNotFound, isJetStreamMissing, isStreamNotFound } from "./jetstream-missing.js";
|
|
26
24
|
import { assertValidKind } from "./agent-file.js";
|
|
27
25
|
import { assertValidName } from "./resolve.js";
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
30
|
-
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, } from "@nats-io/jetstream";
|
|
26
|
+
import { anycastStoreDefinition, taskDurableConfig } from "./streams.js";
|
|
27
|
+
import { jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, } from "@nats-io/jetstream";
|
|
31
28
|
import {} from "@nats-io/jetstream";
|
|
32
29
|
import { Kvm } from "@nats-io/kv";
|
|
33
30
|
import { listMembers } from "./members.js";
|
|
@@ -41,16 +38,18 @@ import { EndpointSubscriptions, membershipConsumerReleased } from "./endpoint-su
|
|
|
41
38
|
import { EndpointChannelDiscovery } from "./endpoint-channel-discovery.js";
|
|
42
39
|
import { EndpointLiveness } from "./endpoint-liveness.js";
|
|
43
40
|
import { EndpointCredentials } from "./endpoint-credentials.js";
|
|
41
|
+
import { EndpointReconnect } from "./endpoint-reconnect.js";
|
|
42
|
+
import { EndpointConnection } from "./endpoint-connection.js";
|
|
44
43
|
import { EndpointProvisioning } from "./endpoint-provisioning.js";
|
|
45
44
|
import { EndpointBootMemberships } from "./endpoint-boot-memberships.js";
|
|
46
45
|
import { ackWaitOf, durableDelivery, isRecord } from "./endpoint-inbound.js";
|
|
47
|
-
import {
|
|
46
|
+
import { effectiveReplay, effectiveReplayWindowMs, readChannelConfig, readChannelDefaults, } from "./channels.js";
|
|
48
47
|
import { isCasLoss } from "./endpoint-records.js";
|
|
49
48
|
import { ANYCAST_CANCEL_OP, ANYCAST_NOTICE_KEY, ANYCAST_RECORDS_CREDENTIAL_OUTDATED, ANYCAST_STATUS_OP, ANYCAST_TAKES_OP, AnycastRequestAuthority, isAnycastSeq, } from "./anycast-requests.js";
|
|
50
|
-
import { assertValidRole, chatStream, chatSubject, controlServiceSubject, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, membershipInAllow, isConcreteChannel, isChannelFamily, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId,
|
|
49
|
+
import { assertValidRole, chatStream, chatSubject, controlServiceSubject, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, membershipInAllow, isConcreteChannel, isChannelFamily, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
|
|
51
50
|
import { isPermissionDenied } from "./permission-denied.js";
|
|
52
51
|
export { isPermissionDenied };
|
|
53
|
-
import { DEFAULT_SERVER, isPublishPermissionDenied,
|
|
52
|
+
import { DEFAULT_SERVER, isPublishPermissionDenied, isAuthRejection, kvWatchConsumer, decodeBearerPrincipal, RepeatedRefusal, describeStatusError, } from "./endpoint-dial.js";
|
|
54
53
|
export { DEFAULT_SERVER, isPublishPermissionDenied, wsServers, defaultProbeTimeoutMs, dialerFor, isAuthRejection, isReachable, probeConnect, } from "./endpoint-dial.js";
|
|
55
54
|
import { MAX_TIMER_MS, ANYCAST_SWEEP_MS, MAX_TAKES_IN_FLIGHT, ANYCAST_TAKE_TIMEOUT_MS, DmDurableMissingError, DmBindRefusedError, } from "./endpoint-options.js";
|
|
56
55
|
export { deliveryLimitOf, DEFAULT_SPACE, DmDurableMissingError, DmBindRefusedError, fitHistoryPage, } from "./endpoint-options.js";
|
|
@@ -802,461 +801,167 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
802
801
|
throw error;
|
|
803
802
|
}
|
|
804
803
|
}
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
}
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
this.
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
this.
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
// pre-created at `cotal up`; open mode lazily creates it.
|
|
955
|
-
const watchChannels = this.doWatch && this.doWatchChannels;
|
|
956
|
-
if (watchChannels || this.doConsume) {
|
|
957
|
-
this.channelKv = await openChannelRegistry(this.nc, this.space, { create: !this.authed });
|
|
958
|
-
if (watchChannels)
|
|
959
|
-
await (this.readOnce ? this.readChannelsOnce() : this.startChannelWatch());
|
|
960
|
-
// A seat reads the registry on demand, as it reads the roster; this first read fills the cache
|
|
961
|
-
// its boot briefing renders from.
|
|
962
|
-
else if (this.readsChannelsOnDemand)
|
|
963
|
-
await this.readChannels();
|
|
964
|
-
}
|
|
965
|
-
// Rebind the daemon's lease-loss watch onto the fresh connection (#1596). The intent survived
|
|
966
|
-
// clearConnectionScoped; a bind that completes after a stop or another rebuild finds the epoch
|
|
967
|
-
// moved and retires itself. The replay re-delivers the row's current state, so a change that
|
|
968
|
-
// landed in the null window still reaches the trigger.
|
|
969
|
-
if (this.deliveryLease.hasWatchIntent)
|
|
970
|
-
await this.bindDeliveryLeaseWatch();
|
|
971
|
-
// FAIL BEFORE PRESENCE (SPEC 13.1): an AUTHED endpoint that will register on the roster or
|
|
972
|
-
// bind lifecycle-keyed consumers must hold its launcher-supplied lifecycle uid BEFORE anything
|
|
973
|
-
// makes it visible - a missing uid must never leave a roster ghost that could not bind (false
|
|
974
|
-
// readiness). Non-registering, non-consuming infra endpoints (a provisioner window, the
|
|
975
|
-
// delivery daemon, one-shot CLI probes) bind no lifecycle-keyed names and stay outside the
|
|
976
|
-
// rule; open/token mode self-minted at construction, so this only ever throws for a
|
|
977
|
-
// mis-launched AUTH endpoint.
|
|
978
|
-
if (this.authed && (this.doConsume || this.doRegister))
|
|
979
|
-
this.requireLifecycleUid(this.doConsume ? "an authed consuming endpoint" : "an authed presence-registering endpoint");
|
|
980
|
-
// Consumers bind BEFORE presence publishes: the durable bind is the broker's proof that this
|
|
981
|
-
// incarnation's lifecycle-keyed names match its minted grants, so a wrong-uid launch dies
|
|
982
|
-
// with NO presence ghost instead of advertising an agent that can never receive.
|
|
983
|
-
if (this.doConsume) {
|
|
984
|
-
this.jsm = await jetstreamManager(this.nc);
|
|
985
|
-
// Open mode: lazily create the streams on the first endpoint. Auth mode: they are
|
|
986
|
-
// pre-created at `cotal up` and STREAM.CREATE is denied to agents, so skip.
|
|
987
|
-
if (!this.authed)
|
|
988
|
-
await this.ensureStreams();
|
|
989
|
-
await this.startConsumers();
|
|
990
|
-
}
|
|
991
|
-
// The register-only broker proof: a consuming agent proved its lifecycle by binding a
|
|
992
|
-
// durable above, but an authed AGENT that only REGISTERS presence (consume:false) has bound
|
|
993
|
-
// nothing yet, so a wrong-but-valid uid would grammar-pass requireLifecycleUid and then
|
|
994
|
-
// advertise a ghost that can receive nothing. Prove the incarnation at the broker FIRST — an
|
|
995
|
-
// exact info on this agent's own pre-provisioned dm_<owner>-<actor>-<uid> durable (the
|
|
996
|
-
// manager/self-provision minted it under the SAME uid). A wrong uid names a durable that
|
|
997
|
-
// does not exist, so the info throws and presence never publishes. Only for `kind: "agent"`
|
|
998
|
-
// (a peer on the roster); pure `kind: "endpoint"` infra (manager, delivery, feedback intake)
|
|
999
|
-
// holds no dm_ durable and stays exempt.
|
|
1000
|
-
if (this.doRegister && !this.doConsume && this.authed && this.card.kind === "agent") {
|
|
1001
|
-
const jsm = this.jsm ?? (this.jsm = await jetstreamManager(this.nc));
|
|
1002
|
-
const uid = this.requireLifecycleUid("an authed presence-registering agent");
|
|
1003
|
-
try {
|
|
1004
|
-
await jsm.consumers.info(dmStream(this.space), dmDurable(this.owner, this.actor, uid));
|
|
1005
|
-
}
|
|
1006
|
-
catch (e) {
|
|
1007
|
-
throw new Error(`lifecycle proof failed for ${this.card.id} (uid ${uid}): its dm_ durable is not present at the broker, so this incarnation's uid does not match its provisioned resources - refusing to publish presence (SPEC 13.1 fail-before-presence): ${e?.message ?? String(e)}`);
|
|
1008
|
-
}
|
|
1009
|
-
}
|
|
1010
|
-
if (this.doRegister) {
|
|
1011
|
-
// #1356: this await is bounded already — a refused presence write returns no response at all,
|
|
1012
|
-
// so it surfaces as the JetStream request TIMEOUT (~5s), once, with no retry here. What it did
|
|
1013
|
-
// NOT do was say what failed: the bare `timeout` names no bucket, no space and no subsystem,
|
|
1014
|
-
// and a launcher that catches it has nothing to act on. The lifecycle-proof failure a few lines
|
|
1015
|
-
// above throws a fully explanatory message; this one is held to the same standard. Still fails
|
|
1016
|
-
// rather than degrading — SPEC 13.1 fail-before-presence means a registering agent that cannot
|
|
1017
|
-
// publish presence must not come up as though it had.
|
|
1018
|
-
try {
|
|
1019
|
-
await this.publishPresence();
|
|
1020
|
-
}
|
|
1021
|
-
catch (e) {
|
|
1022
|
-
const detail = e?.message ?? String(e);
|
|
1023
|
-
throw new Error(`presence registration failed for ${this.card.id}: the space's presence KV bucket "${presenceBucket(this.space)}" did not accept this endpoint's first write (${detail}). The bucket can be OPENED and WATCHED while refusing every write, so a healthy-looking connection does not rule this out; a broker whose store has latched refuses until it is restarted, and no client can clear it (#1356).`);
|
|
1024
|
-
}
|
|
1025
|
-
// Beat at this endpoint's own phase. A repeating timer re-arms from the instant its callback
|
|
1026
|
-
// ran, and every timer run in one timers pass re-arms to the same instant, so the heartbeats of
|
|
1027
|
-
// many endpoints in one process that once coincided stay coincident and the loop takes the whole
|
|
1028
|
-
// batch in one tick (5088 endpoints in one process: 1 to 2 s stalls, with the put deadlines
|
|
1029
|
-
// expiring inside them). Re-arming from the scheduled beat keeps each endpoint's phase.
|
|
1030
|
-
const phase = Date.now();
|
|
1031
|
-
let beat = 0;
|
|
1032
|
-
const arm = () => {
|
|
1033
|
-
const next = nextHeartbeat(phase, this.heartbeatMs, beat, Date.now());
|
|
1034
|
-
beat = next.beat;
|
|
1035
|
-
const handle = setTimeout(() => {
|
|
1036
|
-
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
1037
|
-
if (this.heartbeatTimer === handle)
|
|
1038
|
-
arm();
|
|
1039
|
-
}, next.delayMs);
|
|
1040
|
-
this.heartbeatTimer = handle;
|
|
1041
|
-
};
|
|
1042
|
-
arm();
|
|
1043
|
-
}
|
|
1044
|
-
// Caller-owned membership watches are INTENT rather than one-connection iterators. Re-open them
|
|
1045
|
-
// before reporting the endpoint connected, so a successful reconnect does not leave the graph stale.
|
|
1046
|
-
await this.rearmMembershipWatches();
|
|
1047
|
-
// Re-arm Plane-3 (delivery-daemon-hosted fan-out + trusted reader + ctl.delivery) on every (re)connect — no-op unless this
|
|
1048
|
-
// endpoint hosts it. The first arm comes from startPlane3 (after start()); this re-binds the loops
|
|
1049
|
-
// a reconnect's clearConnectionScoped() tore down, so a broker blip doesn't silently kill the backstop.
|
|
1050
|
-
await this.armPlane3();
|
|
1051
|
-
// Re-bind the liveness responders on this connection, for armPlane3's reason: the rebuild closed
|
|
1052
|
-
// the old connection and the responders with it.
|
|
1053
|
-
for (const plane of this.livenessResponders.keys())
|
|
1054
|
-
this.bindLiveness(plane);
|
|
1055
|
-
// The principal serves subscribe again on the new connection and run their check again.
|
|
1056
|
-
this.rebindPrincipalServes(false);
|
|
1057
|
-
for (const tap of this.taps)
|
|
1058
|
-
this.bindTap(tap);
|
|
1059
|
-
// Bound and live — covers initial start, manual reconnect, AND background self-heal (every
|
|
1060
|
-
// path lands here). The single signal an in-process agent's connected flag tracks.
|
|
1061
|
-
//
|
|
1062
|
-
// The stopped guard: stop() can land in any await above. Both callers tear the fresh
|
|
1063
|
-
// connection back down (tearDownIfStopped), but an event has no undo, so a late
|
|
1064
|
-
// `connection: true` would be the last edge a listener ever sees on a stopped endpoint and
|
|
1065
|
-
// nothing follows it to correct the record. It belongs here rather than in a consumer because
|
|
1066
|
-
// every listener reads the same edge; MeshAgent carries its own `stopping` guard and so was
|
|
1067
|
-
// never the one exposed, which is the point.
|
|
1068
|
-
//
|
|
1069
|
-
// Measured for the start() caller by the broker suite's mid-bind cell. doRebuild is
|
|
1070
|
-
// also measured, by a separate cell that holds the rebuild's connectAndBind at
|
|
1071
|
-
// armPlane3 after a successful first bind and lands stop() in that window (#1028).
|
|
1072
|
-
// Shared-line reasoning is no longer the rebuild proof.
|
|
1073
|
-
if (this.stopped)
|
|
1074
|
-
return;
|
|
1075
|
-
this.emit("connection", { connected: true });
|
|
1076
|
-
if (this.nc) {
|
|
1077
|
-
for (const follower of this.activeGoalFollowers) {
|
|
1078
|
-
follower.reconnected(this.nc);
|
|
1079
|
-
}
|
|
1080
|
-
}
|
|
804
|
+
/** The connection bind and teardown of this endpoint. Every dependency is read when it is used, so a
|
|
805
|
+
* field assigned after construction is seen. */
|
|
806
|
+
connection = new EndpointConnection({
|
|
807
|
+
bearerSource: () => this.bearerSource,
|
|
808
|
+
currentBearer: () => this.currentBearer,
|
|
809
|
+
bearerRenewalFailures: () => this.bearerRenewalFailures,
|
|
810
|
+
bearerAdoptedAtMs: () => this.bearerAdoptedAtMs,
|
|
811
|
+
credsSource: () => this.credsSource,
|
|
812
|
+
currentCreds: () => this.currentCreds,
|
|
813
|
+
userMode: () => this.userMode,
|
|
814
|
+
bearerRefusalSurfaced: () => this.bearerRefusalSurfaced,
|
|
815
|
+
setBearerRefusalSurfaced: (value) => { this.bearerRefusalSurfaced = value; },
|
|
816
|
+
nc: () => this.nc,
|
|
817
|
+
setNc: (value) => { this.nc = value; },
|
|
818
|
+
servers: () => this.servers,
|
|
819
|
+
connId: () => this.connId,
|
|
820
|
+
card: () => this.card,
|
|
821
|
+
transportPingIntervalMs: () => this.transportPingIntervalMs,
|
|
822
|
+
transportMaxPingOut: () => this.transportMaxPingOut,
|
|
823
|
+
transportConnectTimeoutMs: () => this.transportConnectTimeoutMs,
|
|
824
|
+
token: () => this.token,
|
|
825
|
+
user: () => this.user,
|
|
826
|
+
pass: () => this.pass,
|
|
827
|
+
sentinelCreds: () => this.sentinelCreds,
|
|
828
|
+
tls: () => this.tls,
|
|
829
|
+
watchDeletePermissions: () => this.watchDeletePermissions,
|
|
830
|
+
ownedConnections: () => this.ownedConnections,
|
|
831
|
+
onConnection: () => this.onConnection,
|
|
832
|
+
js: () => this.js,
|
|
833
|
+
setJs: (value) => { this.js = value; },
|
|
834
|
+
dmOnly: () => this.dmOnly,
|
|
835
|
+
beforeBind: () => this.beforeBind,
|
|
836
|
+
stopped: () => this.stopped,
|
|
837
|
+
setStopped: (value) => { this.stopped = value; },
|
|
838
|
+
acceptedToken: () => this.acceptedToken,
|
|
839
|
+
space: () => this.space,
|
|
840
|
+
owner: () => this.owner,
|
|
841
|
+
actor: () => this.actor,
|
|
842
|
+
issuedGeneration: () => this.issuedGeneration,
|
|
843
|
+
setIssuedGeneration: (value) => { this.issuedGeneration = value; },
|
|
844
|
+
doWatch: () => this.doWatch,
|
|
845
|
+
doRegister: () => this.doRegister,
|
|
846
|
+
kv: () => this.kv,
|
|
847
|
+
setKv: (value) => { this.kv = value; },
|
|
848
|
+
authed: () => this.authed,
|
|
849
|
+
ttlMs: () => this.ttlMs,
|
|
850
|
+
readOnce: () => this.readOnce,
|
|
851
|
+
sweepTimer: () => this.sweepTimer,
|
|
852
|
+
setSweepTimer: (value) => { this.sweepTimer = value; },
|
|
853
|
+
doWatchChannels: () => this.doWatchChannels,
|
|
854
|
+
doConsume: () => this.doConsume,
|
|
855
|
+
channelKv: () => this.channelKv,
|
|
856
|
+
setChannelKv: (value) => { this.channelKv = value; },
|
|
857
|
+
readsChannelsOnDemand: () => this.readsChannelsOnDemand,
|
|
858
|
+
deliveryLease: () => this.deliveryLease,
|
|
859
|
+
jsm: () => this.jsm,
|
|
860
|
+
setJsm: (value) => { this.jsm = value; },
|
|
861
|
+
heartbeatMs: () => this.heartbeatMs,
|
|
862
|
+
heartbeatTimer: () => this.heartbeatTimer,
|
|
863
|
+
setHeartbeatTimer: (value) => { this.heartbeatTimer = value; },
|
|
864
|
+
livenessResponders: () => this.livenessResponders,
|
|
865
|
+
taps: () => this.taps,
|
|
866
|
+
activeGoalFollowers: () => this.activeGoalFollowers,
|
|
867
|
+
authExpiryReconnectTimer: () => this.authExpiryReconnectTimer,
|
|
868
|
+
setAuthExpiryReconnectTimer: (value) => { this.authExpiryReconnectTimer = value; },
|
|
869
|
+
streamMsgs: () => this.streamMsgs,
|
|
870
|
+
presenceEpoch: () => this.presenceEpoch,
|
|
871
|
+
setPresenceEpoch: (value) => { this.presenceEpoch = value; },
|
|
872
|
+
presence: () => this.presence,
|
|
873
|
+
channelWatchIter: () => this.channelWatchIter,
|
|
874
|
+
setChannelWatchIter: (value) => { this.channelWatchIter = value; },
|
|
875
|
+
chatSubs: () => this.chatSubs,
|
|
876
|
+
chatSubDenied: () => this.chatSubDenied,
|
|
877
|
+
ownWatchDeletePrefixes: () => this.ownWatchDeletePrefixes,
|
|
878
|
+
channelConfigs: () => this.channelConfigs,
|
|
879
|
+
channelDefaults: () => this.channelDefaults,
|
|
880
|
+
setChannelDefaults: (value) => { this.channelDefaults = value; },
|
|
881
|
+
channelRead: () => this.channelRead,
|
|
882
|
+
setChannelRead: (value) => { this.channelRead = value; },
|
|
883
|
+
channelReadGeneration: () => this.channelReadGeneration,
|
|
884
|
+
setChannelReadGeneration: (value) => { this.channelReadGeneration = value; },
|
|
885
|
+
channelReadApplied: () => this.channelReadApplied,
|
|
886
|
+
setChannelReadApplied: (value) => { this.channelReadApplied = value; },
|
|
887
|
+
membershipFeedWatches: () => this.membershipFeedWatches,
|
|
888
|
+
membersKv: () => this.membersKv,
|
|
889
|
+
setMembersKv: (value) => { this.membersKv = value; },
|
|
890
|
+
aclKv: () => this.aclKv,
|
|
891
|
+
setAclKv: (value) => { this.aclKv = value; },
|
|
892
|
+
membershipFeedKv: () => this.membershipFeedKv,
|
|
893
|
+
setMembershipFeedKv: (value) => { this.membershipFeedKv = value; },
|
|
894
|
+
managerLeaseKv: () => this.managerLeaseKv,
|
|
895
|
+
setManagerLeaseKv: (value) => { this.managerLeaseKv = value; },
|
|
896
|
+
daemonRenewalLeaseRevision: () => this.daemonRenewalLeaseRevision,
|
|
897
|
+
setDaemonRenewalLeaseRevision: (value) => { this.daemonRenewalLeaseRevision = value; },
|
|
898
|
+
deliveryPlane: () => this.deliveryPlane,
|
|
899
|
+
subs: () => this.subs,
|
|
900
|
+
principalServes: () => this.principalServes,
|
|
901
|
+
membershipPauses: () => this.membershipPauses,
|
|
902
|
+
cancelBearerRenewal: () => this.cancelBearerRenewal,
|
|
903
|
+
credsTimer: () => this.credsTimer,
|
|
904
|
+
dmOnlyInFlight: () => this.dmOnlyInFlight,
|
|
905
|
+
rebuildPromise: () => this.rebuildPromise,
|
|
906
|
+
startPromise: () => this.startPromise,
|
|
907
|
+
reconnecting: () => this.reconnecting,
|
|
908
|
+
credsTxnsRunning: () => this.credsTxnsRunning,
|
|
909
|
+
residentSwap: () => this.residentSwap,
|
|
910
|
+
nativeBound: () => this.nativeBound,
|
|
911
|
+
refreshBearer: (initial) => this.refreshBearer(initial),
|
|
912
|
+
refreshCreds: (initial) => this.refreshCreds(initial),
|
|
913
|
+
bearerForWire: () => this.bearerForWire(),
|
|
914
|
+
presentCreds: () => this.presentCreds(),
|
|
915
|
+
presentBearer: () => this.presentBearer(),
|
|
916
|
+
armAuthExpiryReconnectFence: (nc) => this.armAuthExpiryReconnectFence(nc),
|
|
917
|
+
watchStatus: () => this.watchStatus(),
|
|
918
|
+
bindDmOnly: (mode) => this.bindDmOnly(mode),
|
|
919
|
+
requireLifecycleUid: (what) => this.requireLifecycleUid(what),
|
|
920
|
+
readPresenceOnce: () => this.readPresenceOnce(),
|
|
921
|
+
startPresenceWatch: () => this.startPresenceWatch(),
|
|
922
|
+
sweep: () => this.sweep(),
|
|
923
|
+
readChannelsOnce: () => this.readChannelsOnce(),
|
|
924
|
+
startChannelWatch: () => this.startChannelWatch(),
|
|
925
|
+
readChannels: () => this.readChannels(),
|
|
926
|
+
bindDeliveryLeaseWatch: () => this.bindDeliveryLeaseWatch(),
|
|
927
|
+
ensureStreams: () => this.ensureStreams(),
|
|
928
|
+
startConsumers: () => this.startConsumers(),
|
|
929
|
+
publishPresence: () => this.publishPresence(),
|
|
930
|
+
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
931
|
+
rearmMembershipWatches: () => this.rearmMembershipWatches(),
|
|
932
|
+
armPlane3: () => this.armPlane3(),
|
|
933
|
+
bindLiveness: (plane) => this.bindLiveness(plane),
|
|
934
|
+
rebindPrincipalServes: (reconnected) => this.rebindPrincipalServes(reconnected),
|
|
935
|
+
bindTap: (tap) => this.bindTap(tap),
|
|
936
|
+
retireKvWatch: (iter) => this.retireKvWatch(iter),
|
|
937
|
+
stopDeliveryLeaseWatch: () => this.stopDeliveryLeaseWatch(),
|
|
938
|
+
disarmMembershipWatch: (watch) => this.disarmMembershipWatch(watch),
|
|
939
|
+
disableLibraryReconnect: (nc) => this.disableLibraryReconnect(nc),
|
|
940
|
+
dropAnycastRequests: () => this.dropAnycastRequests(),
|
|
941
|
+
closeWithoutLibraryReconnect: (nc) => this.closeWithoutLibraryReconnect(nc),
|
|
942
|
+
stopPrincipalServe: (intent, republish) => this.stopPrincipalServe(intent, republish),
|
|
943
|
+
kickBackoff: () => this.kickBackoff(),
|
|
944
|
+
deleteRetainedMembershipConsumer: (watch) => this.deleteRetainedMembershipConsumer(watch),
|
|
945
|
+
settleDeliveryLeaseWatches: () => this.settleDeliveryLeaseWatches(),
|
|
946
|
+
settleKvWatches: () => this.settleKvWatches(),
|
|
947
|
+
clearPresenceWriteFailure: () => this.clearPresenceWriteFailure(),
|
|
948
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
949
|
+
});
|
|
950
|
+
/** Forwards to the connection component's {@link EndpointConnection.bindConnection}. */
|
|
951
|
+
bindConnection() {
|
|
952
|
+
return this.connection.bindConnection();
|
|
1081
953
|
}
|
|
1082
|
-
/**
|
|
1083
|
-
* second heartbeat, double-pump a consumer, or keep stale roster ghosts. Caller-owned
|
|
1084
|
-
* subs (tap/serve) are left alone — they aren't rebuilt here. */
|
|
954
|
+
/** Forwards to the connection component's {@link EndpointConnection.clearConnectionScoped}. */
|
|
1085
955
|
clearConnectionScoped() {
|
|
1086
|
-
|
|
1087
|
-
clearTimeout(this.authExpiryReconnectTimer);
|
|
1088
|
-
this.authExpiryReconnectTimer = undefined;
|
|
1089
|
-
}
|
|
1090
|
-
if (this.heartbeatTimer) {
|
|
1091
|
-
clearInterval(this.heartbeatTimer);
|
|
1092
|
-
this.heartbeatTimer = undefined;
|
|
1093
|
-
}
|
|
1094
|
-
if (this.sweepTimer) {
|
|
1095
|
-
clearInterval(this.sweepTimer);
|
|
1096
|
-
this.sweepTimer = undefined;
|
|
1097
|
-
}
|
|
1098
|
-
for (const msgs of this.streamMsgs) {
|
|
1099
|
-
try {
|
|
1100
|
-
msgs.stop();
|
|
1101
|
-
}
|
|
1102
|
-
catch {
|
|
1103
|
-
/* already closed with the connection */
|
|
1104
|
-
}
|
|
1105
|
-
}
|
|
1106
|
-
this.streamMsgs.length = 0;
|
|
1107
|
-
this.presenceEpoch++;
|
|
1108
|
-
this.presence.cancelRebind();
|
|
1109
|
-
this.presence.retireWatch();
|
|
1110
|
-
if (this.channelWatchIter)
|
|
1111
|
-
this.retireKvWatch(this.channelWatchIter);
|
|
1112
|
-
this.channelWatchIter = undefined;
|
|
1113
|
-
this.stopDeliveryLeaseWatch();
|
|
1114
|
-
for (const sub of this.chatSubs.values()) {
|
|
1115
|
-
try {
|
|
1116
|
-
sub.unsubscribe();
|
|
1117
|
-
}
|
|
1118
|
-
catch {
|
|
1119
|
-
/* already closed with the connection */
|
|
1120
|
-
}
|
|
1121
|
-
}
|
|
1122
|
-
this.chatSubs.clear();
|
|
1123
|
-
this.chatSubDenied.clear();
|
|
1124
|
-
this.ownWatchDeletePrefixes.clear();
|
|
1125
|
-
this.presence.resetConnection();
|
|
1126
|
-
this.channelConfigs.clear();
|
|
1127
|
-
this.channelDefaults = {};
|
|
1128
|
-
// A read in flight on the old connection must not land in the new connection's cache.
|
|
1129
|
-
this.channelRead = undefined;
|
|
1130
|
-
this.channelReadGeneration++;
|
|
1131
|
-
this.channelReadApplied = this.channelReadGeneration;
|
|
1132
|
-
for (const watch of this.membershipFeedWatches)
|
|
1133
|
-
watch.arm = watch.arm.catch(() => { }).then(() => this.disarmMembershipWatch(watch)).catch((err) => { this.emit("error", err); });
|
|
1134
|
-
}
|
|
1135
|
-
/** A transport can connect before a later JetStream/KV/subscription bind fails. The caller will
|
|
1136
|
-
* retry the whole start, so release that partial epoch first instead of overwriting `nc` and
|
|
1137
|
-
* leaving one established socket behind per retry. `close()` is deliberate: a failed bind has no
|
|
1138
|
-
* graceful delivery contract to drain, and cleanup itself must not extend the retry failure.
|
|
1139
|
-
*
|
|
1140
|
-
* A throwing `close()` is not hypothetical: nats.js `NatsConnectionImpl.close()` awaits the
|
|
1141
|
-
* protocol close, which itself flushes outbound + tears down subscription state; any of those
|
|
1142
|
-
* that reject leaves the underlying TCP transport ESTABLISHED even though every client handle
|
|
1143
|
-
* on this endpoint has been cleared. The next `start()` on the same object then dials a fresh
|
|
1144
|
-
* connection while the old socket stays live on the broker (`/varz.connections` climbs once per
|
|
1145
|
-
* failed attempt). The recovery path is layered, from most graceful to most surgical, matching
|
|
1146
|
-
* nats-core's public surface (nats.d.ts: `close`, `drain`, `isClosed`; protocol.d.ts owns the
|
|
1147
|
-
* `transport` handle whose `close(err?)` is the socket teardown; transport.d.ts exposes a
|
|
1148
|
-
* synchronous `disconnect()` for the case where even close rejects). We only touch a private
|
|
1149
|
-
* (`nc.protocol.transport`) when the public API has already refused, and the ORIGINAL bind
|
|
1150
|
-
* error is re-raised at every layer. The close error is diagnostic noise. The bind error is
|
|
1151
|
-
* what the caller must see.
|
|
1152
|
-
*
|
|
1153
|
-
* The delivery-serve subs and `this.subs` array are handles created by a successful
|
|
1154
|
-
* `armDeliveryControl` on a PREVIOUS epoch; a bind failure on the retry never rewrites them
|
|
1155
|
-
* because arming is a late step. Left in place they point at a dead protocol and survive into
|
|
1156
|
-
* the next retry. The reviewer flagged this as an untested handle class. Zero them here so a
|
|
1157
|
-
* retry starts with the same empty state as a first attempt. */
|
|
1158
|
-
async closeFailedBind() {
|
|
1159
|
-
const failedNc = this.nc;
|
|
1160
|
-
if (failedNc)
|
|
1161
|
-
this.disableLibraryReconnect(failedNc);
|
|
1162
|
-
this.clearConnectionScoped();
|
|
1163
|
-
this.nc = undefined;
|
|
1164
|
-
this.js = undefined;
|
|
1165
|
-
this.jsm = undefined;
|
|
1166
|
-
this.kv = undefined;
|
|
1167
|
-
this.channelKv = undefined;
|
|
1168
|
-
this.membersKv = undefined;
|
|
1169
|
-
this.aclKv = undefined;
|
|
1170
|
-
this.membershipFeedKv = undefined;
|
|
1171
|
-
this.deliveryLease.resetConnection();
|
|
1172
|
-
this.managerLeaseKv = undefined;
|
|
1173
|
-
this.daemonRenewalLeaseRevision = undefined;
|
|
1174
|
-
// Handles that armDeliveryControl created on a previous connection: null them so a rebind
|
|
1175
|
-
// does not carry a dead protocol's subs forward. Best-effort unsubscribe first (the sub is
|
|
1176
|
-
// dead with its connection either way; unsubscribing an already-dead sub is a noop).
|
|
1177
|
-
if (this.deliveryPlane.deliveryServeSub) {
|
|
1178
|
-
try {
|
|
1179
|
-
this.deliveryPlane.deliveryServeSub.unsubscribe();
|
|
1180
|
-
}
|
|
1181
|
-
catch { /* dead with the old connection */ }
|
|
1182
|
-
this.deliveryPlane.deliveryServeSub = undefined;
|
|
1183
|
-
}
|
|
1184
|
-
if (this.deliveryPlane.deliveryAdminServeSub) {
|
|
1185
|
-
try {
|
|
1186
|
-
this.deliveryPlane.deliveryAdminServeSub.unsubscribe();
|
|
1187
|
-
}
|
|
1188
|
-
catch { /* dead with the old connection */ }
|
|
1189
|
-
this.deliveryPlane.deliveryAdminServeSub = undefined;
|
|
1190
|
-
}
|
|
1191
|
-
this.dropAnycastRequests();
|
|
1192
|
-
for (const sub of this.subs) {
|
|
1193
|
-
try {
|
|
1194
|
-
sub.unsubscribe();
|
|
1195
|
-
}
|
|
1196
|
-
catch { /* dead with the old connection */ }
|
|
1197
|
-
}
|
|
1198
|
-
this.subs.length = 0;
|
|
1199
|
-
if (!failedNc)
|
|
1200
|
-
return;
|
|
1201
|
-
// The old status iterator is stale after nc is cleared, so its close cannot report this edge.
|
|
1202
|
-
// As in doRebuild, teardown itself must announce the no-nc window before closing the socket.
|
|
1203
|
-
this.emit("transport", { connected: false });
|
|
1204
|
-
this.emit("connection", { connected: false });
|
|
1205
|
-
// Layered teardown, comments kept OUT of the code span below on purpose: the mutation fixture
|
|
1206
|
-
// bin/smoke/mutations/failed-bind-cleanup.json anchors on that span verbatim and the fixture
|
|
1207
|
-
// census (bin/smoke/mutation-fixtures.smoke.ts) reddens an anchor that crosses a comment line.
|
|
1208
|
-
// Fast path: the public close resolved, the broker drops the connection on its own.
|
|
1209
|
-
// Layer 1: nats.js exposes drain() as the sanctioned "close on error" path. It flushes
|
|
1210
|
-
// outbound then closes, and unlike close() has its own idempotent guard against a half-closed
|
|
1211
|
-
// protocol. If drain resolves the socket is gone.
|
|
1212
|
-
// Layer 2: last resort. nats.js `NatsConnectionImpl.protocol.transport` (see
|
|
1213
|
-
// node_modules/@nats-io/nats-core/lib/{nats,protocol,transport}.d.ts) is the TCP socket the
|
|
1214
|
-
// dialer produced. Its `close(err?)` returns a Promise once the socket is torn down; its
|
|
1215
|
-
// synchronous `disconnect()` forces the FIN without waiting. Reached only after the public
|
|
1216
|
-
// API has already refused, guarded only against the shape not being what the pinned version
|
|
1217
|
-
// documents.
|
|
1218
|
-
try {
|
|
1219
|
-
await failedNc.close();
|
|
1220
|
-
return;
|
|
1221
|
-
}
|
|
1222
|
-
catch { /* fall through to the layered fallback */ }
|
|
1223
|
-
if (!failedNc.isClosed?.()) {
|
|
1224
|
-
try {
|
|
1225
|
-
await drainAndClose(failedNc);
|
|
1226
|
-
return;
|
|
1227
|
-
}
|
|
1228
|
-
catch { /* both graceful paths refused, drop to the transport */ }
|
|
1229
|
-
}
|
|
1230
|
-
const proto = failedNc.protocol;
|
|
1231
|
-
const transport = proto?.transport;
|
|
1232
|
-
if (transport && !transport.isClosed) {
|
|
1233
|
-
try {
|
|
1234
|
-
await transport.close?.();
|
|
1235
|
-
}
|
|
1236
|
-
catch { /* transport close itself threw, force it down synchronously below */ }
|
|
1237
|
-
if (!transport.isClosed) {
|
|
1238
|
-
try {
|
|
1239
|
-
transport.disconnect?.();
|
|
1240
|
-
}
|
|
1241
|
-
catch { /* nothing left to try */ }
|
|
1242
|
-
}
|
|
1243
|
-
}
|
|
956
|
+
return this.connection.clearConnectionScoped();
|
|
1244
957
|
}
|
|
1245
|
-
/**
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
return false;
|
|
1253
|
-
const nc = this.nc;
|
|
1254
|
-
if (nc)
|
|
1255
|
-
this.disableLibraryReconnect(nc);
|
|
1256
|
-
this.clearConnectionScoped();
|
|
1257
|
-
await this.closeWithoutLibraryReconnect(nc);
|
|
1258
|
-
this.nc = undefined;
|
|
1259
|
-
return true;
|
|
958
|
+
/** Forwards to the connection component's {@link EndpointConnection.closeFailedBind}. */
|
|
959
|
+
closeFailedBind() {
|
|
960
|
+
return this.connection.closeFailedBind();
|
|
961
|
+
}
|
|
962
|
+
/** Forwards to the connection component's {@link EndpointConnection.tearDownIfStopped}. */
|
|
963
|
+
tearDownIfStopped() {
|
|
964
|
+
return this.connection.tearDownIfStopped();
|
|
1260
965
|
}
|
|
1261
966
|
/** Watch for a terminal close (nats.js has exhausted its own reconnect) and rebuild.
|
|
1262
967
|
* Our own stop()/drain also resolves closed(), so the `stopped` guard keeps a clean
|
|
@@ -1281,95 +986,68 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1281
986
|
void this.reestablishLoop();
|
|
1282
987
|
});
|
|
1283
988
|
}
|
|
1284
|
-
/**
|
|
1285
|
-
*
|
|
1286
|
-
|
|
1287
|
-
|
|
989
|
+
/** The rebuild and retry pacing of this endpoint. Every dependency is read when it is used, so a field
|
|
990
|
+
* assigned after construction is seen. */
|
|
991
|
+
reconnection = new EndpointReconnect({
|
|
992
|
+
rebuildPromise: () => this.rebuildPromise,
|
|
993
|
+
setRebuildPromise: (value) => { this.rebuildPromise = value; },
|
|
994
|
+
nc: () => this.nc,
|
|
995
|
+
setNc: (value) => { this.nc = value; },
|
|
996
|
+
reconnecting: () => this.reconnecting,
|
|
997
|
+
setReconnecting: (value) => { this.reconnecting = value; },
|
|
998
|
+
membershipFeedWatches: () => this.membershipFeedWatches,
|
|
999
|
+
js: () => this.js,
|
|
1000
|
+
setJs: (value) => { this.js = value; },
|
|
1001
|
+
jsm: () => this.jsm,
|
|
1002
|
+
setJsm: (value) => { this.jsm = value; },
|
|
1003
|
+
kv: () => this.kv,
|
|
1004
|
+
setKv: (value) => { this.kv = value; },
|
|
1005
|
+
channelKv: () => this.channelKv,
|
|
1006
|
+
setChannelKv: (value) => { this.channelKv = value; },
|
|
1007
|
+
membersKv: () => this.membersKv,
|
|
1008
|
+
setMembersKv: (value) => { this.membersKv = value; },
|
|
1009
|
+
aclKv: () => this.aclKv,
|
|
1010
|
+
setAclKv: (value) => { this.aclKv = value; },
|
|
1011
|
+
membershipFeedKv: () => this.membershipFeedKv,
|
|
1012
|
+
setMembershipFeedKv: (value) => { this.membershipFeedKv = value; },
|
|
1013
|
+
deliveryLease: () => this.deliveryLease,
|
|
1014
|
+
managerLeaseKv: () => this.managerLeaseKv,
|
|
1015
|
+
setManagerLeaseKv: (value) => { this.managerLeaseKv = value; },
|
|
1016
|
+
daemonRenewalLeaseRevision: () => this.daemonRenewalLeaseRevision,
|
|
1017
|
+
setDaemonRenewalLeaseRevision: (value) => { this.daemonRenewalLeaseRevision = value; },
|
|
1018
|
+
stopped: () => this.stopped,
|
|
1019
|
+
retryMs: () => this.retryMs,
|
|
1020
|
+
retryAttempt: () => this.retryAttempt,
|
|
1021
|
+
setRetryAttempt: (value) => { this.retryAttempt = value; },
|
|
1022
|
+
backoffResolve: () => this.backoffResolve,
|
|
1023
|
+
backoffTimer: () => this.backoffTimer,
|
|
1024
|
+
setBackoffTimer: (value) => { this.backoffTimer = value; },
|
|
1025
|
+
membershipPauses: () => this.membershipPauses,
|
|
1026
|
+
disableLibraryReconnect: (nc) => this.disableLibraryReconnect(nc),
|
|
1027
|
+
clearConnectionScoped: () => this.clearConnectionScoped(),
|
|
1028
|
+
settleDeliveryLeaseWatches: () => this.settleDeliveryLeaseWatches(),
|
|
1029
|
+
settleKvWatches: () => this.settleKvWatches(),
|
|
1030
|
+
closeWithoutLibraryReconnect: (nc) => this.closeWithoutLibraryReconnect(nc),
|
|
1031
|
+
connectAndBind: () => this.connectAndBind(),
|
|
1032
|
+
tearDownIfStopped: () => this.tearDownIfStopped(),
|
|
1033
|
+
superviseConnection: () => this.superviseConnection(),
|
|
1034
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
1035
|
+
});
|
|
1036
|
+
/** Forwards to the reconnect component's {@link EndpointReconnect.rebuild}. */
|
|
1288
1037
|
rebuild() {
|
|
1289
|
-
|
|
1290
|
-
return this.rebuildPromise;
|
|
1291
|
-
const p = this.doRebuild().finally(() => {
|
|
1292
|
-
if (this.rebuildPromise === p)
|
|
1293
|
-
this.rebuildPromise = undefined;
|
|
1294
|
-
});
|
|
1295
|
-
this.rebuildPromise = p;
|
|
1296
|
-
return p;
|
|
1297
|
-
}
|
|
1298
|
-
/** The transition: stop the connection-scoped timers FIRST (so nothing live touches
|
|
1299
|
-
* this.nc during the null window), drop the connection refs, close the old nc, then
|
|
1300
|
-
* rebind + re-arm the supervisor on the fresh connection. clearConnectionScoped is
|
|
1301
|
-
* idempotent, so connectAndBind's own call here is a noop. */
|
|
1302
|
-
async doRebuild() {
|
|
1303
|
-
const oldNc = this.nc;
|
|
1304
|
-
if (oldNc)
|
|
1305
|
-
this.disableLibraryReconnect(oldNc);
|
|
1306
|
-
this.reconnecting = true;
|
|
1307
|
-
try {
|
|
1308
|
-
this.clearConnectionScoped();
|
|
1309
|
-
// Manual reconnect still has a live old epoch: complete broker-consumer cleanup before close.
|
|
1310
|
-
// Terminal self-heal has an already-closed epoch: disarm retains stream/name for fresh cleanup,
|
|
1311
|
-
// and the KV watch consumers that closed with it are left to the broker's inactive threshold.
|
|
1312
|
-
if (oldNc && !oldNc.isClosed())
|
|
1313
|
-
await Promise.all([...[...this.membershipFeedWatches].map((watch) => watch.arm), this.settleDeliveryLeaseWatches(), this.settleKvWatches()]);
|
|
1314
|
-
this.nc = undefined;
|
|
1315
|
-
this.js = undefined;
|
|
1316
|
-
this.jsm = undefined;
|
|
1317
|
-
this.kv = undefined;
|
|
1318
|
-
this.channelKv = undefined;
|
|
1319
|
-
// Plane-3 KV handles are bound to the old connection too — drop them so the daemon re-opens them on
|
|
1320
|
-
// the fresh nc (else durableJoin/leave/list, the reader's ACL re-auth, and lease renew use a dead
|
|
1321
|
-
// handle after a reconnect).
|
|
1322
|
-
this.membersKv = undefined;
|
|
1323
|
-
this.aclKv = undefined;
|
|
1324
|
-
this.membershipFeedKv = undefined;
|
|
1325
|
-
this.deliveryLease.resetConnection();
|
|
1326
|
-
// The manager's liveness-lease handle too: left bound to the old connection, every renew and
|
|
1327
|
-
// re-read after a reconnect times out, and the manager reports its lease unknown for good.
|
|
1328
|
-
this.managerLeaseKv = undefined;
|
|
1329
|
-
this.daemonRenewalLeaseRevision = undefined;
|
|
1330
|
-
// This is an application-requested epoch teardown, not a transient nats.js blip. The old
|
|
1331
|
-
// status iterator is now stale by construction and its close is epoch-dropped, so this line is
|
|
1332
|
-
// the authoritative raw-liveness edge for the no-nc window until the new watcher seeds true.
|
|
1333
|
-
this.emit("transport", { connected: false });
|
|
1334
|
-
this.emit("connection", { connected: false });
|
|
1335
|
-
await this.closeWithoutLibraryReconnect(oldNc);
|
|
1336
|
-
// stop() can arrive while the old epoch closes. Do not start another authentication
|
|
1337
|
-
// handshake after it: a terminal credential may already have expired in that window.
|
|
1338
|
-
// Keep old-epoch and late-epoch close failures outside this dial-only catch.
|
|
1339
|
-
if (this.stopped)
|
|
1340
|
-
return;
|
|
1341
|
-
try {
|
|
1342
|
-
await this.connectAndBind();
|
|
1343
|
-
}
|
|
1344
|
-
catch (error) {
|
|
1345
|
-
if (!this.stopped)
|
|
1346
|
-
throw error;
|
|
1347
|
-
}
|
|
1348
|
-
// stop() may have run during the await — don't leave a live connection + heartbeat +
|
|
1349
|
-
// supervisor on a stopped endpoint. (Reads this.nc in its own scope — a bare `this.nc`
|
|
1350
|
-
// here in doRebuild narrows to `never` via TS inlining connectAndBind's assignment.)
|
|
1351
|
-
// Re-arm on the fresh nc only after this stopped fence accepts it.
|
|
1352
|
-
if (await this.tearDownIfStopped())
|
|
1353
|
-
return;
|
|
1354
|
-
this.superviseConnection();
|
|
1355
|
-
}
|
|
1356
|
-
finally {
|
|
1357
|
-
this.reconnecting = false;
|
|
1358
|
-
}
|
|
1038
|
+
return this.reconnection.rebuild();
|
|
1359
1039
|
}
|
|
1360
|
-
/**
|
|
1361
|
-
* a standing one (a down broker, a credential nothing can renew), and retrying it every few
|
|
1362
|
-
* seconds forever is what turns one stuck client into a permanent flat load on the broker and its
|
|
1363
|
-
* auth callout. */
|
|
1364
|
-
static RETRY_BACKOFF_CAP_MS = 60_000;
|
|
1365
|
-
/** The wait before the next rebuild attempt: {@link retryMs}, doubling per consecutive failure up
|
|
1366
|
-
* to {@link RETRY_BACKOFF_CAP_MS}, then a random half of it taken off. A transient drop still
|
|
1367
|
-
* recovers within retryMs, and endpoints refused together, as a broker restart or an auth limit
|
|
1368
|
-
* refuses a host's agents, spread their retries instead of arriving at the callout at once. */
|
|
1040
|
+
/** Forwards to the reconnect component's {@link EndpointReconnect.nextRetryDelayMs}. */
|
|
1369
1041
|
nextRetryDelayMs() {
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1042
|
+
return this.reconnection.nextRetryDelayMs();
|
|
1043
|
+
}
|
|
1044
|
+
/** Forwards to the reconnect component's {@link EndpointReconnect.kickBackoff}. */
|
|
1045
|
+
kickBackoff() {
|
|
1046
|
+
return this.reconnection.kickBackoff();
|
|
1047
|
+
}
|
|
1048
|
+
/** Forwards to the reconnect component's {@link EndpointReconnect.membershipPause}. */
|
|
1049
|
+
membershipPause(attempt) {
|
|
1050
|
+
return this.reconnection.membershipPause(attempt);
|
|
1373
1051
|
}
|
|
1374
1052
|
/** Rebuild with backoff until it sticks or we're stopped. Interruptible: a manual
|
|
1375
1053
|
* {@link reconnect} kicks the backoff so the next attempt runs immediately instead of
|
|
@@ -1419,30 +1097,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1419
1097
|
if (!this.stopped && this.reestablishing && !this.rebuildPromise)
|
|
1420
1098
|
this.kickBackoff();
|
|
1421
1099
|
}
|
|
1422
|
-
/** Cut an in-flight reestablish backoff short so the next attempt runs immediately, and
|
|
1423
|
-
* clear its timer so it can't fire later on a stopped/restarted loop. */
|
|
1424
|
-
kickBackoff() {
|
|
1425
|
-
this.backoffResolve?.();
|
|
1426
|
-
if (this.backoffTimer) {
|
|
1427
|
-
clearTimeout(this.backoffTimer);
|
|
1428
|
-
this.backoffTimer = undefined;
|
|
1429
|
-
}
|
|
1430
|
-
}
|
|
1431
|
-
/** Capped exponential pause between membership retries. stop() ends it early, so a loop waiting
|
|
1432
|
-
* on an absent delivery daemon never holds a stopped endpoint's process open for the delay. */
|
|
1433
|
-
membershipPause(attempt) {
|
|
1434
|
-
if (this.stopped)
|
|
1435
|
-
return Promise.resolve();
|
|
1436
|
-
return new Promise((resolve) => {
|
|
1437
|
-
const wake = () => {
|
|
1438
|
-
clearTimeout(timer);
|
|
1439
|
-
this.membershipPauses.delete(wake);
|
|
1440
|
-
resolve();
|
|
1441
|
-
};
|
|
1442
|
-
const timer = setTimeout(wake, Math.min(30_000, 1000 * 2 ** attempt));
|
|
1443
|
-
this.membershipPauses.add(wake);
|
|
1444
|
-
});
|
|
1445
|
-
}
|
|
1446
1100
|
/** Manual reconnect: tear down the current connection and rebuild, WITHOUT the permanent
|
|
1447
1101
|
* stop (stopped/stopping stay false). Serialized with the self-heal supervisor via
|
|
1448
1102
|
* {@link rebuild}, and interruptible — if a backoff is in flight, kick it so the attempt
|
|
@@ -1474,99 +1128,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1474
1128
|
stop() {
|
|
1475
1129
|
return this.stopPromise ??= this.stopOnce();
|
|
1476
1130
|
}
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
this.stopped = true;
|
|
1481
|
-
for (const follower of this.activeGoalFollowers) {
|
|
1482
|
-
follower.cancel();
|
|
1483
|
-
}
|
|
1484
|
-
this.activeGoalFollowers.clear();
|
|
1485
|
-
this.presenceEpoch++;
|
|
1486
|
-
// Every principal serve ends now, so a `ready` still waiting for the broker rejects and no new
|
|
1487
|
-
// request is taken. Its handlers in flight finish below, before the connection they reply on
|
|
1488
|
-
// closes, or their requests' deadlines pass.
|
|
1489
|
-
const principalStops = [...this.principalServes.values()].map((intent) => this.stopPrincipalServe(intent, false));
|
|
1490
|
-
this.presence.cancelRebind();
|
|
1491
|
-
// Wake a reestablishLoop sitting in backoff so it sees `stopped` and exits instead of
|
|
1492
|
-
// sleeping out retryMs; also clears the timer so it can't fire later.
|
|
1493
|
-
this.kickBackoff();
|
|
1494
|
-
for (const wake of [...this.membershipPauses])
|
|
1495
|
-
wake();
|
|
1496
|
-
if (this.heartbeatTimer)
|
|
1497
|
-
clearInterval(this.heartbeatTimer);
|
|
1498
|
-
if (this.sweepTimer)
|
|
1499
|
-
clearInterval(this.sweepTimer);
|
|
1500
|
-
this.cancelBearerRenewal?.();
|
|
1501
|
-
if (this.credsTimer)
|
|
1502
|
-
clearTimeout(this.credsTimer);
|
|
1503
|
-
if (this.authExpiryReconnectTimer)
|
|
1504
|
-
clearTimeout(this.authExpiryReconnectTimer);
|
|
1505
|
-
for (const watch of this.membershipFeedWatches) {
|
|
1506
|
-
watch.stopped = true;
|
|
1507
|
-
watch.arm = watch.arm.catch(() => { }).then(async () => {
|
|
1508
|
-
await this.disarmMembershipWatch(watch);
|
|
1509
|
-
await this.deleteRetainedMembershipConsumer(watch);
|
|
1510
|
-
});
|
|
1511
|
-
}
|
|
1512
|
-
// Permanent endpoint shutdown has no future epoch. Try strict cleanup on the current live epoch;
|
|
1513
|
-
// if the broker is already gone, terminate local ownership rather than hanging shutdown forever.
|
|
1514
|
-
await Promise.all([...this.membershipFeedWatches].map((watch) => watch.arm.catch((err) => {
|
|
1515
|
-
if (this.nc && !this.nc.isClosed())
|
|
1516
|
-
throw err;
|
|
1517
|
-
})));
|
|
1518
|
-
for (const watch of this.membershipFeedWatches) {
|
|
1519
|
-
watch.resolveStop?.();
|
|
1520
|
-
watch.resolveStop = undefined;
|
|
1521
|
-
watch.rejectStop = undefined;
|
|
1522
|
-
}
|
|
1523
|
-
this.membershipFeedWatches.clear();
|
|
1524
|
-
for (const msgs of this.streamMsgs) {
|
|
1525
|
-
try {
|
|
1526
|
-
msgs.stop();
|
|
1527
|
-
}
|
|
1528
|
-
catch {
|
|
1529
|
-
/* already closed */
|
|
1530
|
-
}
|
|
1531
|
-
}
|
|
1532
|
-
this.presence.retireWatch();
|
|
1533
|
-
if (this.channelWatchIter)
|
|
1534
|
-
this.retireKvWatch(this.channelWatchIter);
|
|
1535
|
-
this.channelWatchIter = undefined;
|
|
1536
|
-
this.stopDeliveryLeaseWatch();
|
|
1537
|
-
await Promise.all(principalStops);
|
|
1538
|
-
try {
|
|
1539
|
-
if (this.doRegister) {
|
|
1540
|
-
await this.presence.publishOffline();
|
|
1541
|
-
}
|
|
1542
|
-
}
|
|
1543
|
-
catch {
|
|
1544
|
-
/* best-effort graceful leave */
|
|
1545
|
-
}
|
|
1546
|
-
await Promise.all([this.settleDeliveryLeaseWatches(), this.settleKvWatches()]);
|
|
1547
|
-
// DM-only: the pull above is stopped, so no new copy reaches the handler. Let a running handler
|
|
1548
|
-
// settle while the connection is still open, so its ack can reach the broker; close either way.
|
|
1549
|
-
let dmOnlyUnsettled = false;
|
|
1550
|
-
if (this.dmOnly && this.dmOnlyInFlight) {
|
|
1551
|
-
const settleMs = this.dmOnly.settleTimeoutMs ?? 30_000;
|
|
1552
|
-
dmOnlyUnsettled = await withDeadline(this.dmOnlyInFlight.then(() => false), settleMs, () => new Error("unsettled"))
|
|
1553
|
-
.catch(() => true);
|
|
1554
|
-
}
|
|
1555
|
-
await this.closeWithoutLibraryReconnect(this.nc);
|
|
1556
|
-
// #1356: stop() does its own teardown and never runs clearConnectionScoped, so clear here too —
|
|
1557
|
-
// AFTER the best-effort offline publish above, which would otherwise re-record a refusal on its
|
|
1558
|
-
// way out. A stopped endpoint has no live basis for "this bucket is refusing writes", and the
|
|
1559
|
-
// duration is computed from `Date.now()`, so a retained record does not merely go stale: it keeps
|
|
1560
|
-
// COUNTING UP for as long as the dead object is held.
|
|
1561
|
-
this.clearPresenceWriteFailure();
|
|
1562
|
-
// Custody callers must not observe stop completion while a concurrent rebind can still
|
|
1563
|
-
// publish a late consuming connection. doRebuild closes its stopped epoch before resolving.
|
|
1564
|
-
await this.rebuildPromise;
|
|
1565
|
-
await this.startPromise;
|
|
1566
|
-
if ([...this.ownedConnections].some((nc) => !nc.isClosed()))
|
|
1567
|
-
throw new Error("endpoint stop could not prove its connection closed; local consumer custody must be retained");
|
|
1568
|
-
if (dmOnlyUnsettled)
|
|
1569
|
-
throw new Error(`endpoint stop closed the connection but the DM-only handler did not settle within ${this.dmOnly?.settleTimeoutMs ?? 30_000}ms; its copy stays unacked and handler custody must be retained`);
|
|
1131
|
+
/** Forwards to the connection component's {@link EndpointConnection.stopOnce}. */
|
|
1132
|
+
stopOnce() {
|
|
1133
|
+
return this.connection.stopOnce();
|
|
1570
1134
|
}
|
|
1571
1135
|
/** The native binding of every connection this endpoint holds. A host compares the list with a
|
|
1572
1136
|
* complete native census of the account to show that no other connection uses it. The list
|
|
@@ -1575,20 +1139,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1575
1139
|
* start, during a start, rebuild or stop, while a credential transaction or its connection swap
|
|
1576
1140
|
* is pending, and while the connection is down or redialing. */
|
|
1577
1141
|
nativeConnections() {
|
|
1578
|
-
|
|
1579
|
-
if (this.startPromise !== undefined || this.rebuildPromise !== undefined || this.reconnecting)
|
|
1580
|
-
throw unsettled("a start or rebuild is in flight");
|
|
1581
|
-
const open = [...this.ownedConnections].filter((nc) => !nc.isClosed());
|
|
1582
|
-
if (this.stopped) {
|
|
1583
|
-
if (open.length > 0)
|
|
1584
|
-
throw unsettled("the endpoint stopped but a connection has not closed");
|
|
1585
|
-
return [];
|
|
1586
|
-
}
|
|
1587
|
-
if (this.credsTxnsRunning > 0 || this.residentSwap !== undefined)
|
|
1588
|
-
throw unsettled("a credential adoption is in flight");
|
|
1589
|
-
if (!this.nc || !this.nativeBound)
|
|
1590
|
-
throw unsettled("the connection is down or redialing");
|
|
1591
|
-
return open.map((nc) => nativeConnectionBinding(nc, "endpoint connection"));
|
|
1142
|
+
return this.connection.nativeConnections();
|
|
1592
1143
|
}
|
|
1593
1144
|
// ---- messaging -----------------------------------------------------------
|
|
1594
1145
|
/** The messaging verbs of this endpoint: multicast, unicast, anycast and the event-payload
|