@cotal-ai/core 0.171.3 → 0.171.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/control-exchange.d.ts.map +1 -1
- package/dist/control-exchange.js +3 -4
- package/dist/control-exchange.js.map +1 -1
- package/dist/endpoint-anycast.d.ts +110 -0
- package/dist/endpoint-anycast.d.ts.map +1 -0
- package/dist/endpoint-anycast.js +316 -0
- package/dist/endpoint-anycast.js.map +1 -0
- package/dist/endpoint-connection.d.ts +363 -0
- package/dist/endpoint-connection.d.ts.map +1 -0
- package/dist/endpoint-connection.js +741 -0
- package/dist/endpoint-connection.js.map +1 -0
- package/dist/endpoint-consumers.d.ts +147 -0
- package/dist/endpoint-consumers.d.ts.map +1 -0
- package/dist/endpoint-consumers.js +459 -0
- package/dist/endpoint-consumers.js.map +1 -0
- package/dist/endpoint-dial.d.ts +3 -1
- package/dist/endpoint-dial.d.ts.map +1 -1
- package/dist/endpoint-dial.js +5 -2
- package/dist/endpoint-dial.js.map +1 -1
- package/dist/endpoint-durable.d.ts +82 -0
- package/dist/endpoint-durable.d.ts.map +1 -0
- package/dist/endpoint-durable.js +164 -0
- package/dist/endpoint-durable.js.map +1 -0
- package/dist/endpoint-principal-serve.d.ts +110 -0
- package/dist/endpoint-principal-serve.d.ts.map +1 -0
- package/dist/endpoint-principal-serve.js +233 -0
- package/dist/endpoint-principal-serve.js.map +1 -0
- package/dist/endpoint-reconnect.d.ts +129 -0
- package/dist/endpoint-reconnect.d.ts.map +1 -0
- package/dist/endpoint-reconnect.js +166 -0
- package/dist/endpoint-reconnect.js.map +1 -0
- package/dist/endpoint.d.ts +44 -241
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +403 -1740
- package/dist/endpoint.js.map +1 -1
- package/dist/launch-artifacts.d.ts.map +1 -1
- package/dist/launch-artifacts.js +17 -9
- package/dist/launch-artifacts.js.map +1 -1
- package/dist/websocket-dial.d.ts +9 -0
- package/dist/websocket-dial.d.ts.map +1 -0
- package/dist/websocket-dial.js +121 -0
- package/dist/websocket-dial.js.map +1 -0
- package/package.json +1 -1
package/dist/endpoint.js
CHANGED
|
@@ -1,65 +1,61 @@
|
|
|
1
1
|
import { EndpointHistory } from "./endpoint-history.js";
|
|
2
2
|
export { MULTI_FILTER_BATCH, MULTI_FILTER_READ_CONCURRENCY } from "./endpoint-history.js";
|
|
3
|
-
import { nativeBindingAfter
|
|
3
|
+
import { nativeBindingAfter } from "./native-connection.js";
|
|
4
4
|
import { EventEmitter } from "node:events";
|
|
5
|
-
import { drainAndClose } from "./connection-cleanup.js";
|
|
6
5
|
import { randomUUID } from "node:crypto";
|
|
7
|
-
import {
|
|
6
|
+
import { PermissionViolationError, RequestError, TimeoutError, } from "@nats-io/transport-node";
|
|
8
7
|
import { responderFromLease, responderFromProbe, } from "./liveness.js";
|
|
9
8
|
import { EndpointPresence } from "./endpoint-presence.js";
|
|
10
9
|
export { PresenceWriteStuckError } from "./endpoint-presence.js";
|
|
11
|
-
import {
|
|
12
|
-
import { requireBrokerFloor } from "./broker-floor.js";
|
|
13
|
-
import { withDeadline } from "./deadline.js";
|
|
10
|
+
import { idFromCreds } from "./identity.js";
|
|
14
11
|
import {} from "./secret-store.js";
|
|
15
12
|
import { BIND_SPLIT_REISSUES, resolveService } from "./endpoint-invoke.js";
|
|
16
13
|
import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTargetUnmapped } from "./endpoint-envelope.js";
|
|
17
14
|
import { isRepeatSafeCommand } from "./endpoint-grants.js";
|
|
18
15
|
import { assertGeneration, assertRuntimeToken, endpointToken } from "./endpoint-subjects.js";
|
|
19
|
-
import {
|
|
16
|
+
import {} from "./endpoint-serve.js";
|
|
20
17
|
import { epPrincipalCall } from "./endpoint-call.js";
|
|
21
|
-
import { AGENT_CLUSTER,
|
|
18
|
+
import { AGENT_CLUSTER, AGENT_ENDPOINT } from "./agent-endpoint.js";
|
|
22
19
|
import { processRuntime } from "./process-runtime.js";
|
|
23
|
-
import {
|
|
20
|
+
import { issuedPermitsSubject } from "./issued-authority.js";
|
|
24
21
|
import { listRegistryEndpoints } from "./registry-client.js";
|
|
25
|
-
import {
|
|
22
|
+
import { isJetStreamMissing } from "./jetstream-missing.js";
|
|
26
23
|
import { assertValidKind } from "./agent-file.js";
|
|
27
24
|
import { assertValidName } from "./resolve.js";
|
|
28
|
-
import {
|
|
29
|
-
import { anycastStoreDefinition, taskDurableConfig, PRESENCE_STORAGE } from "./streams.js";
|
|
30
|
-
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, } from "@nats-io/jetstream";
|
|
25
|
+
import { JetStreamApiCodes, } from "@nats-io/jetstream";
|
|
31
26
|
import {} from "@nats-io/jetstream";
|
|
32
|
-
import {
|
|
27
|
+
import {} from "@nats-io/kv";
|
|
33
28
|
import { listMembers } from "./members.js";
|
|
34
|
-
import { readAcl } from "./acls.js";
|
|
35
29
|
import { EndpointDeliveryLease } from "./endpoint-delivery-lease.js";
|
|
36
30
|
import { EndpointDeliveryPlane } from "./endpoint-delivery-plane.js";
|
|
37
31
|
import { EndpointControlPlane } from "./endpoint-control-plane.js";
|
|
38
32
|
import { EndpointMessaging, assertPartsSerializable } from "./endpoint-messaging.js";
|
|
39
33
|
import { EndpointDurableBackstop } from "./endpoint-durable-backstop.js";
|
|
34
|
+
import { EndpointDurable } from "./endpoint-durable.js";
|
|
35
|
+
import { EndpointAnycast } from "./endpoint-anycast.js";
|
|
36
|
+
import { EndpointConsumers } from "./endpoint-consumers.js";
|
|
37
|
+
import { EndpointPrincipalServe } from "./endpoint-principal-serve.js";
|
|
40
38
|
import { EndpointSubscriptions, membershipConsumerReleased } from "./endpoint-subscriptions.js";
|
|
41
39
|
import { EndpointChannelDiscovery } from "./endpoint-channel-discovery.js";
|
|
42
40
|
import { EndpointLiveness } from "./endpoint-liveness.js";
|
|
43
41
|
import { EndpointCredentials } from "./endpoint-credentials.js";
|
|
42
|
+
import { EndpointReconnect } from "./endpoint-reconnect.js";
|
|
43
|
+
import { EndpointConnection } from "./endpoint-connection.js";
|
|
44
44
|
import { EndpointProvisioning } from "./endpoint-provisioning.js";
|
|
45
45
|
import { EndpointBootMemberships } from "./endpoint-boot-memberships.js";
|
|
46
|
-
import {
|
|
47
|
-
import {
|
|
46
|
+
import { isRecord } from "./endpoint-inbound.js";
|
|
47
|
+
import { effectiveReplay, effectiveReplayWindowMs, readChannelConfig, readChannelDefaults, } from "./channels.js";
|
|
48
48
|
import { isCasLoss } from "./endpoint-records.js";
|
|
49
|
-
import {
|
|
50
|
-
import { assertValidRole, chatStream,
|
|
49
|
+
import { AnycastRequestAuthority, } from "./anycast-requests.js";
|
|
50
|
+
import { assertValidRole, chatStream, controlServiceSubject, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, isConcreteChannel, isChannelFamily, parseSubject, assertInboxConnId, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, DEV_OWNER, spaceWildcard, subjectMatches, unicastSubject, unicastRecvFilter, } from "./subjects.js";
|
|
51
51
|
import { isPermissionDenied } from "./permission-denied.js";
|
|
52
52
|
export { isPermissionDenied };
|
|
53
|
-
import { DEFAULT_SERVER, isPublishPermissionDenied,
|
|
53
|
+
import { DEFAULT_SERVER, isPublishPermissionDenied, isAuthRejection, kvWatchConsumer, decodeBearerPrincipal, RepeatedRefusal, describeStatusError, } from "./endpoint-dial.js";
|
|
54
54
|
export { DEFAULT_SERVER, isPublishPermissionDenied, wsServers, defaultProbeTimeoutMs, dialerFor, isAuthRejection, isReachable, probeConnect, } from "./endpoint-dial.js";
|
|
55
|
-
import { MAX_TIMER_MS,
|
|
55
|
+
import { MAX_TIMER_MS, } from "./endpoint-options.js";
|
|
56
56
|
export { deliveryLimitOf, DEFAULT_SPACE, DmDurableMissingError, DmBindRefusedError, fitHistoryPage, } from "./endpoint-options.js";
|
|
57
|
-
import { historyMessageFromDelivery, isUsableMessageId, isFromSubjectSender, authenticatedMessage, isReadableMessagePart,
|
|
57
|
+
import { historyMessageFromDelivery, isUsableMessageId, isFromSubjectSender, authenticatedMessage, isReadableMessagePart, } from "./endpoint-messages.js";
|
|
58
58
|
export { isFromSubjectSender, } from "./endpoint-messages.js";
|
|
59
|
-
/** The highest `prompts` presence `seq` this process used under each runtime id (SPEC §6). A reader
|
|
60
|
-
* orders lists by `seq` within one `runtime.id`, and a runtime id lives as long as the process, so
|
|
61
|
-
* this does too: across a stopped serve, a stopped endpoint and a new endpoint for the lifecycle. */
|
|
62
|
-
const promptsSeqs = new Map();
|
|
63
59
|
/** The principal endpoint each `{space, runtime id, endpoint}` is served by in this process. Two
|
|
64
60
|
* would subscribe the same subject and both answer one request (SPEC 13.21). */
|
|
65
61
|
const principalServers = new Map();
|
|
@@ -189,16 +185,16 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
189
185
|
serveControl: (...args) => this.serveControl(...args),
|
|
190
186
|
aclRegistry: (...args) => this.aclRegistry(...args),
|
|
191
187
|
membersRegistry: (...args) => this.membersRegistry(...args),
|
|
192
|
-
armAnycastRequests: (...args) => this.armAnycastRequests(...args),
|
|
193
|
-
deliveryAnycast: (...args) => this.deliveryAnycast(...args),
|
|
194
|
-
dropAnycastRequests: (...args) => this.dropAnycastRequests(...args),
|
|
188
|
+
armAnycastRequests: (...args) => this.anycastRail.armAnycastRequests(...args),
|
|
189
|
+
deliveryAnycast: (...args) => this.anycastRail.deliveryAnycast(...args),
|
|
190
|
+
dropAnycastRequests: (...args) => this.anycastRail.dropAnycastRequests(...args),
|
|
195
191
|
manager: (...args) => this.manager(...args),
|
|
196
192
|
notLiveMsg: (...args) => this.notLiveMsg(...args),
|
|
197
|
-
plane3MsgId: (...args) => this.plane3MsgId(...args),
|
|
198
|
-
publishDinbox: (...args) => this.publishDinbox(...args),
|
|
199
|
-
resolveOwnerByName: (...args) => this.resolveOwnerByName(...args),
|
|
200
|
-
scheduleResidentSwap: (...args) => this.scheduleResidentSwap(...args),
|
|
201
|
-
sendMentionNotice: (...args) => this.sendMentionNotice(...args),
|
|
193
|
+
plane3MsgId: (...args) => this.durableBackstop.plane3MsgId(...args),
|
|
194
|
+
publishDinbox: (...args) => this.durableBackstop.publishDinbox(...args),
|
|
195
|
+
resolveOwnerByName: (...args) => this.durableBackstop.resolveOwnerByName(...args),
|
|
196
|
+
scheduleResidentSwap: (...args) => this.credentials.scheduleResidentSwap(...args),
|
|
197
|
+
sendMentionNotice: (...args) => this.anycastRail.sendMentionNotice(...args),
|
|
202
198
|
emit: (event, ...args) => this.emit(event, ...args),
|
|
203
199
|
});
|
|
204
200
|
/** The anycast take responder, the request records it writes, and their maintenance timer
|
|
@@ -235,7 +231,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
235
231
|
notLiveMsg: () => this.notLiveMsg(),
|
|
236
232
|
requireLifecycleUid: (operation) => this.requireLifecycleUid(operation),
|
|
237
233
|
requestDelivery: (op, args) => this.requestDelivery(op, args),
|
|
238
|
-
deleteConsumerIfPresent: (stream, name) => this.deleteConsumerIfPresent(stream, name),
|
|
234
|
+
deleteConsumerIfPresent: (stream, name) => this.consumers.deleteConsumerIfPresent(stream, name),
|
|
239
235
|
joinPolicyFresh: (channel) => this.joinPolicyFresh(channel),
|
|
240
236
|
chatMayHaveLost: (subject, sinceSeq) => this.chatMayHaveLost(subject, sinceSeq),
|
|
241
237
|
emit: (event, ...args) => this.emit(event, ...args),
|
|
@@ -253,7 +249,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
253
249
|
* reported itself connected again. */
|
|
254
250
|
taps = [];
|
|
255
251
|
/** The liveness responders {@link serveLiveness} was asked for, by plane. This is INTENT, re-bound
|
|
256
|
-
* by {@link bindLiveness} on every (re)connect: a rebuild closes the old connection and every
|
|
252
|
+
* by {@link EndpointLiveness.bindLiveness} on every (re)connect: a rebuild closes the old connection and every
|
|
257
253
|
* subscription on it, so a responder bound once would leave its plane to the broker's
|
|
258
254
|
* no-responders answer, which a peer grades `unbound`, while this endpoint is connected and serving. */
|
|
259
255
|
livenessResponders = new Map();
|
|
@@ -280,7 +276,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
280
276
|
* so this in-memory mirror is kept, not rebuilt. Cleared only on full stop. */
|
|
281
277
|
plane3Channels = new Map();
|
|
282
278
|
/** Channels whose live sub was REFUSED while they held a Plane-3 durable membership, whose §7
|
|
283
|
-
* tombstone has not yet confirmed (channel → join generation). {@link closeRefusedMembership} retries
|
|
279
|
+
* tombstone has not yet confirmed (channel → join generation). {@link EndpointDurable.closeRefusedMembership} retries
|
|
284
280
|
* the tombstone until it lands; until then this is a `durable-unclosed` state surfaced via
|
|
285
281
|
* {@link pendingDurableLeaves} (the connector shows it in `cotal_channels`, never as ordinary
|
|
286
282
|
* absence). Persists across reconnect; cleared on tombstone success or full stop. */
|
|
@@ -293,7 +289,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
293
289
|
pendingBootJoins = new Set();
|
|
294
290
|
/** The refusals a channel's live subscription callback received. nats.js hands a subscription
|
|
295
291
|
* refusal to the refused subscription's callback and then dispatches that same instance on the
|
|
296
|
-
* connection status, so the callback owns it (see {@link subscribeChat}) and watchStatus skips it. */
|
|
292
|
+
* connection status, so the callback owns it (see {@link EndpointConsumers.subscribeChat}) and watchStatus skips it. */
|
|
297
293
|
chatSubRefusals = new WeakSet();
|
|
298
294
|
/** Service iterators and connection status can receive the same subscription denial. */
|
|
299
295
|
reportedSubRefusals = new WeakSet();
|
|
@@ -341,7 +337,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
341
337
|
/** Bumped by every connection-scoped teardown and by {@link stop}. A presence or lease-watch bind
|
|
342
338
|
* that was awaiting the broker when the epoch moved belongs to a retired epoch: it releases the
|
|
343
339
|
* iterator it got and installs nothing (see {@link startPresenceWatch} and
|
|
344
|
-
* {@link bindDeliveryLeaseWatch}). */
|
|
340
|
+
* {@link EndpointDeliveryLease.bindDeliveryLeaseWatch}). */
|
|
345
341
|
presenceEpoch = 0;
|
|
346
342
|
stopped = false;
|
|
347
343
|
stopPromise;
|
|
@@ -366,7 +362,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
366
362
|
* now instead of awaiting the full retryMs. */
|
|
367
363
|
backoffResolve;
|
|
368
364
|
backoffTimer;
|
|
369
|
-
/** Wakes for the membership retry loops' pauses (see {@link membershipPause}); stop() runs them all. */
|
|
365
|
+
/** Wakes for the membership retry loops' pauses (see {@link EndpointReconnect.membershipPause}); stop() runs them all. */
|
|
370
366
|
membershipPauses = new Set();
|
|
371
367
|
retryMs = 3000;
|
|
372
368
|
/** Consecutive failed rebuilds, driving {@link nextRetryDelayMs}. Reset on every success. */
|
|
@@ -708,18 +704,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
708
704
|
refreshBearer(initial = false) {
|
|
709
705
|
return this.credentials.refreshBearer(initial);
|
|
710
706
|
}
|
|
711
|
-
/** Forwards to the credentials component's {@link EndpointCredentials.bearerForWire}. */
|
|
712
|
-
bearerForWire() {
|
|
713
|
-
return this.credentials.bearerForWire();
|
|
714
|
-
}
|
|
715
|
-
/** Forwards to the credentials component's {@link EndpointCredentials.presentCreds}. */
|
|
716
|
-
presentCreds() {
|
|
717
|
-
return this.credentials.presentCreds();
|
|
718
|
-
}
|
|
719
|
-
/** Forwards to the credentials component's {@link EndpointCredentials.presentBearer}. */
|
|
720
|
-
presentBearer() {
|
|
721
|
-
return this.credentials.presentBearer();
|
|
722
|
-
}
|
|
723
707
|
/** The credential this endpoint presents on its next connect, checked as {@link credsForWire}
|
|
724
708
|
* checks it. A second connection under the same identity reads it on every attempt, so it
|
|
725
709
|
* presents each credential this endpoint adopts and never an expired one. */
|
|
@@ -730,18 +714,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
730
714
|
disableLibraryReconnect(nc) {
|
|
731
715
|
return this.credentials.disableLibraryReconnect(nc);
|
|
732
716
|
}
|
|
733
|
-
/** Forwards to the credentials component's {@link EndpointCredentials.closeWithoutLibraryReconnect}. */
|
|
734
|
-
closeWithoutLibraryReconnect(nc) {
|
|
735
|
-
return this.credentials.closeWithoutLibraryReconnect(nc);
|
|
736
|
-
}
|
|
737
717
|
/** Forwards to the credentials component's {@link EndpointCredentials.armAuthExpiryReconnectFence}. */
|
|
738
718
|
armAuthExpiryReconnectFence(nc) {
|
|
739
719
|
return this.credentials.armAuthExpiryReconnectFence(nc);
|
|
740
720
|
}
|
|
741
|
-
/** Forwards to the credentials component's {@link EndpointCredentials.refreshCreds}. */
|
|
742
|
-
refreshCreds(initial = false) {
|
|
743
|
-
return this.credentials.refreshCreds(initial);
|
|
744
|
-
}
|
|
745
721
|
/** Forwards to the credentials component's {@link EndpointCredentials.renewCredsOnTimer}. */
|
|
746
722
|
renewCredsOnTimer() {
|
|
747
723
|
return this.credentials.renewCredsOnTimer();
|
|
@@ -772,10 +748,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
772
748
|
reloadCreds(expected) {
|
|
773
749
|
return this.credentials.reloadCreds(expected);
|
|
774
750
|
}
|
|
775
|
-
/** Forwards to the credentials component's {@link EndpointCredentials.scheduleResidentSwap}. */
|
|
776
|
-
scheduleResidentSwap() {
|
|
777
|
-
return this.credentials.scheduleResidentSwap();
|
|
778
|
-
}
|
|
779
751
|
armCredsRefresh(delayMs) {
|
|
780
752
|
if (this.stopped)
|
|
781
753
|
return;
|
|
@@ -791,7 +763,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
791
763
|
}
|
|
792
764
|
/** Open the connection and bind everything that hangs off it: status watch, presence
|
|
793
765
|
* watch + heartbeat, channel registry, and the durable consumers. Re-runnable — a
|
|
794
|
-
* reconnect calls it again after {@link clearConnectionScoped}; every binding is
|
|
766
|
+
* reconnect calls it again after {@link EndpointConnection.clearConnectionScoped}; every binding is
|
|
795
767
|
* idempotent (durables bind by name, JetStream dedups by msgID, KV opens are idempotent). */
|
|
796
768
|
async connectAndBind() {
|
|
797
769
|
try {
|
|
@@ -802,461 +774,163 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
802
774
|
throw error;
|
|
803
775
|
}
|
|
804
776
|
}
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
}
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
this.
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
this.
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
// pre-created at `cotal up`; open mode lazily creates it.
|
|
955
|
-
const watchChannels = this.doWatch && this.doWatchChannels;
|
|
956
|
-
if (watchChannels || this.doConsume) {
|
|
957
|
-
this.channelKv = await openChannelRegistry(this.nc, this.space, { create: !this.authed });
|
|
958
|
-
if (watchChannels)
|
|
959
|
-
await (this.readOnce ? this.readChannelsOnce() : this.startChannelWatch());
|
|
960
|
-
// A seat reads the registry on demand, as it reads the roster; this first read fills the cache
|
|
961
|
-
// its boot briefing renders from.
|
|
962
|
-
else if (this.readsChannelsOnDemand)
|
|
963
|
-
await this.readChannels();
|
|
964
|
-
}
|
|
965
|
-
// Rebind the daemon's lease-loss watch onto the fresh connection (#1596). The intent survived
|
|
966
|
-
// clearConnectionScoped; a bind that completes after a stop or another rebuild finds the epoch
|
|
967
|
-
// moved and retires itself. The replay re-delivers the row's current state, so a change that
|
|
968
|
-
// landed in the null window still reaches the trigger.
|
|
969
|
-
if (this.deliveryLease.hasWatchIntent)
|
|
970
|
-
await this.bindDeliveryLeaseWatch();
|
|
971
|
-
// FAIL BEFORE PRESENCE (SPEC 13.1): an AUTHED endpoint that will register on the roster or
|
|
972
|
-
// bind lifecycle-keyed consumers must hold its launcher-supplied lifecycle uid BEFORE anything
|
|
973
|
-
// makes it visible - a missing uid must never leave a roster ghost that could not bind (false
|
|
974
|
-
// readiness). Non-registering, non-consuming infra endpoints (a provisioner window, the
|
|
975
|
-
// delivery daemon, one-shot CLI probes) bind no lifecycle-keyed names and stay outside the
|
|
976
|
-
// rule; open/token mode self-minted at construction, so this only ever throws for a
|
|
977
|
-
// mis-launched AUTH endpoint.
|
|
978
|
-
if (this.authed && (this.doConsume || this.doRegister))
|
|
979
|
-
this.requireLifecycleUid(this.doConsume ? "an authed consuming endpoint" : "an authed presence-registering endpoint");
|
|
980
|
-
// Consumers bind BEFORE presence publishes: the durable bind is the broker's proof that this
|
|
981
|
-
// incarnation's lifecycle-keyed names match its minted grants, so a wrong-uid launch dies
|
|
982
|
-
// with NO presence ghost instead of advertising an agent that can never receive.
|
|
983
|
-
if (this.doConsume) {
|
|
984
|
-
this.jsm = await jetstreamManager(this.nc);
|
|
985
|
-
// Open mode: lazily create the streams on the first endpoint. Auth mode: they are
|
|
986
|
-
// pre-created at `cotal up` and STREAM.CREATE is denied to agents, so skip.
|
|
987
|
-
if (!this.authed)
|
|
988
|
-
await this.ensureStreams();
|
|
989
|
-
await this.startConsumers();
|
|
990
|
-
}
|
|
991
|
-
// The register-only broker proof: a consuming agent proved its lifecycle by binding a
|
|
992
|
-
// durable above, but an authed AGENT that only REGISTERS presence (consume:false) has bound
|
|
993
|
-
// nothing yet, so a wrong-but-valid uid would grammar-pass requireLifecycleUid and then
|
|
994
|
-
// advertise a ghost that can receive nothing. Prove the incarnation at the broker FIRST — an
|
|
995
|
-
// exact info on this agent's own pre-provisioned dm_<owner>-<actor>-<uid> durable (the
|
|
996
|
-
// manager/self-provision minted it under the SAME uid). A wrong uid names a durable that
|
|
997
|
-
// does not exist, so the info throws and presence never publishes. Only for `kind: "agent"`
|
|
998
|
-
// (a peer on the roster); pure `kind: "endpoint"` infra (manager, delivery, feedback intake)
|
|
999
|
-
// holds no dm_ durable and stays exempt.
|
|
1000
|
-
if (this.doRegister && !this.doConsume && this.authed && this.card.kind === "agent") {
|
|
1001
|
-
const jsm = this.jsm ?? (this.jsm = await jetstreamManager(this.nc));
|
|
1002
|
-
const uid = this.requireLifecycleUid("an authed presence-registering agent");
|
|
1003
|
-
try {
|
|
1004
|
-
await jsm.consumers.info(dmStream(this.space), dmDurable(this.owner, this.actor, uid));
|
|
1005
|
-
}
|
|
1006
|
-
catch (e) {
|
|
1007
|
-
throw new Error(`lifecycle proof failed for ${this.card.id} (uid ${uid}): its dm_ durable is not present at the broker, so this incarnation's uid does not match its provisioned resources - refusing to publish presence (SPEC 13.1 fail-before-presence): ${e?.message ?? String(e)}`);
|
|
1008
|
-
}
|
|
1009
|
-
}
|
|
1010
|
-
if (this.doRegister) {
|
|
1011
|
-
// #1356: this await is bounded already — a refused presence write returns no response at all,
|
|
1012
|
-
// so it surfaces as the JetStream request TIMEOUT (~5s), once, with no retry here. What it did
|
|
1013
|
-
// NOT do was say what failed: the bare `timeout` names no bucket, no space and no subsystem,
|
|
1014
|
-
// and a launcher that catches it has nothing to act on. The lifecycle-proof failure a few lines
|
|
1015
|
-
// above throws a fully explanatory message; this one is held to the same standard. Still fails
|
|
1016
|
-
// rather than degrading — SPEC 13.1 fail-before-presence means a registering agent that cannot
|
|
1017
|
-
// publish presence must not come up as though it had.
|
|
1018
|
-
try {
|
|
1019
|
-
await this.publishPresence();
|
|
1020
|
-
}
|
|
1021
|
-
catch (e) {
|
|
1022
|
-
const detail = e?.message ?? String(e);
|
|
1023
|
-
throw new Error(`presence registration failed for ${this.card.id}: the space's presence KV bucket "${presenceBucket(this.space)}" did not accept this endpoint's first write (${detail}). The bucket can be OPENED and WATCHED while refusing every write, so a healthy-looking connection does not rule this out; a broker whose store has latched refuses until it is restarted, and no client can clear it (#1356).`);
|
|
1024
|
-
}
|
|
1025
|
-
// Beat at this endpoint's own phase. A repeating timer re-arms from the instant its callback
|
|
1026
|
-
// ran, and every timer run in one timers pass re-arms to the same instant, so the heartbeats of
|
|
1027
|
-
// many endpoints in one process that once coincided stay coincident and the loop takes the whole
|
|
1028
|
-
// batch in one tick (5088 endpoints in one process: 1 to 2 s stalls, with the put deadlines
|
|
1029
|
-
// expiring inside them). Re-arming from the scheduled beat keeps each endpoint's phase.
|
|
1030
|
-
const phase = Date.now();
|
|
1031
|
-
let beat = 0;
|
|
1032
|
-
const arm = () => {
|
|
1033
|
-
const next = nextHeartbeat(phase, this.heartbeatMs, beat, Date.now());
|
|
1034
|
-
beat = next.beat;
|
|
1035
|
-
const handle = setTimeout(() => {
|
|
1036
|
-
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
1037
|
-
if (this.heartbeatTimer === handle)
|
|
1038
|
-
arm();
|
|
1039
|
-
}, next.delayMs);
|
|
1040
|
-
this.heartbeatTimer = handle;
|
|
1041
|
-
};
|
|
1042
|
-
arm();
|
|
1043
|
-
}
|
|
1044
|
-
// Caller-owned membership watches are INTENT rather than one-connection iterators. Re-open them
|
|
1045
|
-
// before reporting the endpoint connected, so a successful reconnect does not leave the graph stale.
|
|
1046
|
-
await this.rearmMembershipWatches();
|
|
1047
|
-
// Re-arm Plane-3 (delivery-daemon-hosted fan-out + trusted reader + ctl.delivery) on every (re)connect — no-op unless this
|
|
1048
|
-
// endpoint hosts it. The first arm comes from startPlane3 (after start()); this re-binds the loops
|
|
1049
|
-
// a reconnect's clearConnectionScoped() tore down, so a broker blip doesn't silently kill the backstop.
|
|
1050
|
-
await this.armPlane3();
|
|
1051
|
-
// Re-bind the liveness responders on this connection, for armPlane3's reason: the rebuild closed
|
|
1052
|
-
// the old connection and the responders with it.
|
|
1053
|
-
for (const plane of this.livenessResponders.keys())
|
|
1054
|
-
this.bindLiveness(plane);
|
|
1055
|
-
// The principal serves subscribe again on the new connection and run their check again.
|
|
1056
|
-
this.rebindPrincipalServes(false);
|
|
1057
|
-
for (const tap of this.taps)
|
|
1058
|
-
this.bindTap(tap);
|
|
1059
|
-
// Bound and live — covers initial start, manual reconnect, AND background self-heal (every
|
|
1060
|
-
// path lands here). The single signal an in-process agent's connected flag tracks.
|
|
1061
|
-
//
|
|
1062
|
-
// The stopped guard: stop() can land in any await above. Both callers tear the fresh
|
|
1063
|
-
// connection back down (tearDownIfStopped), but an event has no undo, so a late
|
|
1064
|
-
// `connection: true` would be the last edge a listener ever sees on a stopped endpoint and
|
|
1065
|
-
// nothing follows it to correct the record. It belongs here rather than in a consumer because
|
|
1066
|
-
// every listener reads the same edge; MeshAgent carries its own `stopping` guard and so was
|
|
1067
|
-
// never the one exposed, which is the point.
|
|
1068
|
-
//
|
|
1069
|
-
// Measured for the start() caller by the broker suite's mid-bind cell. doRebuild is
|
|
1070
|
-
// also measured, by a separate cell that holds the rebuild's connectAndBind at
|
|
1071
|
-
// armPlane3 after a successful first bind and lands stop() in that window (#1028).
|
|
1072
|
-
// Shared-line reasoning is no longer the rebuild proof.
|
|
1073
|
-
if (this.stopped)
|
|
1074
|
-
return;
|
|
1075
|
-
this.emit("connection", { connected: true });
|
|
1076
|
-
if (this.nc) {
|
|
1077
|
-
for (const follower of this.activeGoalFollowers) {
|
|
1078
|
-
follower.reconnected(this.nc);
|
|
1079
|
-
}
|
|
1080
|
-
}
|
|
777
|
+
/** The connection bind and teardown of this endpoint. Every dependency is read when it is used, so a
|
|
778
|
+
* field assigned after construction is seen. */
|
|
779
|
+
connection = new EndpointConnection({
|
|
780
|
+
bearerSource: () => this.bearerSource,
|
|
781
|
+
currentBearer: () => this.currentBearer,
|
|
782
|
+
bearerRenewalFailures: () => this.bearerRenewalFailures,
|
|
783
|
+
bearerAdoptedAtMs: () => this.bearerAdoptedAtMs,
|
|
784
|
+
credsSource: () => this.credsSource,
|
|
785
|
+
currentCreds: () => this.currentCreds,
|
|
786
|
+
userMode: () => this.userMode,
|
|
787
|
+
bearerRefusalSurfaced: () => this.bearerRefusalSurfaced,
|
|
788
|
+
setBearerRefusalSurfaced: (value) => { this.bearerRefusalSurfaced = value; },
|
|
789
|
+
nc: () => this.nc,
|
|
790
|
+
setNc: (value) => { this.nc = value; },
|
|
791
|
+
servers: () => this.servers,
|
|
792
|
+
connId: () => this.connId,
|
|
793
|
+
card: () => this.card,
|
|
794
|
+
transportPingIntervalMs: () => this.transportPingIntervalMs,
|
|
795
|
+
transportMaxPingOut: () => this.transportMaxPingOut,
|
|
796
|
+
transportConnectTimeoutMs: () => this.transportConnectTimeoutMs,
|
|
797
|
+
token: () => this.token,
|
|
798
|
+
user: () => this.user,
|
|
799
|
+
pass: () => this.pass,
|
|
800
|
+
sentinelCreds: () => this.sentinelCreds,
|
|
801
|
+
tls: () => this.tls,
|
|
802
|
+
watchDeletePermissions: () => this.watchDeletePermissions,
|
|
803
|
+
ownedConnections: () => this.ownedConnections,
|
|
804
|
+
onConnection: () => this.onConnection,
|
|
805
|
+
js: () => this.js,
|
|
806
|
+
setJs: (value) => { this.js = value; },
|
|
807
|
+
dmOnly: () => this.dmOnly,
|
|
808
|
+
beforeBind: () => this.beforeBind,
|
|
809
|
+
stopped: () => this.stopped,
|
|
810
|
+
setStopped: (value) => { this.stopped = value; },
|
|
811
|
+
acceptedToken: () => this.acceptedToken,
|
|
812
|
+
space: () => this.space,
|
|
813
|
+
owner: () => this.owner,
|
|
814
|
+
actor: () => this.actor,
|
|
815
|
+
issuedGeneration: () => this.issuedGeneration,
|
|
816
|
+
setIssuedGeneration: (value) => { this.issuedGeneration = value; },
|
|
817
|
+
doWatch: () => this.doWatch,
|
|
818
|
+
doRegister: () => this.doRegister,
|
|
819
|
+
kv: () => this.kv,
|
|
820
|
+
setKv: (value) => { this.kv = value; },
|
|
821
|
+
authed: () => this.authed,
|
|
822
|
+
ttlMs: () => this.ttlMs,
|
|
823
|
+
readOnce: () => this.readOnce,
|
|
824
|
+
sweepTimer: () => this.sweepTimer,
|
|
825
|
+
setSweepTimer: (value) => { this.sweepTimer = value; },
|
|
826
|
+
doWatchChannels: () => this.doWatchChannels,
|
|
827
|
+
doConsume: () => this.doConsume,
|
|
828
|
+
channelKv: () => this.channelKv,
|
|
829
|
+
setChannelKv: (value) => { this.channelKv = value; },
|
|
830
|
+
readsChannelsOnDemand: () => this.readsChannelsOnDemand,
|
|
831
|
+
deliveryLease: () => this.deliveryLease,
|
|
832
|
+
jsm: () => this.jsm,
|
|
833
|
+
setJsm: (value) => { this.jsm = value; },
|
|
834
|
+
heartbeatMs: () => this.heartbeatMs,
|
|
835
|
+
heartbeatTimer: () => this.heartbeatTimer,
|
|
836
|
+
setHeartbeatTimer: (value) => { this.heartbeatTimer = value; },
|
|
837
|
+
livenessResponders: () => this.livenessResponders,
|
|
838
|
+
taps: () => this.taps,
|
|
839
|
+
activeGoalFollowers: () => this.activeGoalFollowers,
|
|
840
|
+
authExpiryReconnectTimer: () => this.authExpiryReconnectTimer,
|
|
841
|
+
setAuthExpiryReconnectTimer: (value) => { this.authExpiryReconnectTimer = value; },
|
|
842
|
+
streamMsgs: () => this.streamMsgs,
|
|
843
|
+
presenceEpoch: () => this.presenceEpoch,
|
|
844
|
+
setPresenceEpoch: (value) => { this.presenceEpoch = value; },
|
|
845
|
+
presence: () => this.presence,
|
|
846
|
+
channelWatchIter: () => this.channelWatchIter,
|
|
847
|
+
setChannelWatchIter: (value) => { this.channelWatchIter = value; },
|
|
848
|
+
chatSubs: () => this.chatSubs,
|
|
849
|
+
chatSubDenied: () => this.chatSubDenied,
|
|
850
|
+
ownWatchDeletePrefixes: () => this.ownWatchDeletePrefixes,
|
|
851
|
+
channelConfigs: () => this.channelConfigs,
|
|
852
|
+
channelDefaults: () => this.channelDefaults,
|
|
853
|
+
setChannelDefaults: (value) => { this.channelDefaults = value; },
|
|
854
|
+
channelRead: () => this.channelRead,
|
|
855
|
+
setChannelRead: (value) => { this.channelRead = value; },
|
|
856
|
+
channelReadGeneration: () => this.channelReadGeneration,
|
|
857
|
+
setChannelReadGeneration: (value) => { this.channelReadGeneration = value; },
|
|
858
|
+
channelReadApplied: () => this.channelReadApplied,
|
|
859
|
+
setChannelReadApplied: (value) => { this.channelReadApplied = value; },
|
|
860
|
+
membershipFeedWatches: () => this.membershipFeedWatches,
|
|
861
|
+
membersKv: () => this.membersKv,
|
|
862
|
+
setMembersKv: (value) => { this.membersKv = value; },
|
|
863
|
+
aclKv: () => this.aclKv,
|
|
864
|
+
setAclKv: (value) => { this.aclKv = value; },
|
|
865
|
+
membershipFeedKv: () => this.membershipFeedKv,
|
|
866
|
+
setMembershipFeedKv: (value) => { this.membershipFeedKv = value; },
|
|
867
|
+
managerLeaseKv: () => this.managerLeaseKv,
|
|
868
|
+
setManagerLeaseKv: (value) => { this.managerLeaseKv = value; },
|
|
869
|
+
daemonRenewalLeaseRevision: () => this.daemonRenewalLeaseRevision,
|
|
870
|
+
setDaemonRenewalLeaseRevision: (value) => { this.daemonRenewalLeaseRevision = value; },
|
|
871
|
+
deliveryPlane: () => this.deliveryPlane,
|
|
872
|
+
subs: () => this.subs,
|
|
873
|
+
principalServes: () => this.principalServes,
|
|
874
|
+
membershipPauses: () => this.membershipPauses,
|
|
875
|
+
cancelBearerRenewal: () => this.cancelBearerRenewal,
|
|
876
|
+
credsTimer: () => this.credsTimer,
|
|
877
|
+
dmOnlyInFlight: () => this.dmOnlyInFlight,
|
|
878
|
+
rebuildPromise: () => this.rebuildPromise,
|
|
879
|
+
startPromise: () => this.startPromise,
|
|
880
|
+
reconnecting: () => this.reconnecting,
|
|
881
|
+
credsTxnsRunning: () => this.credsTxnsRunning,
|
|
882
|
+
residentSwap: () => this.residentSwap,
|
|
883
|
+
nativeBound: () => this.nativeBound,
|
|
884
|
+
refreshBearer: (initial) => this.refreshBearer(initial),
|
|
885
|
+
refreshCreds: (initial) => this.credentials.refreshCreds(initial),
|
|
886
|
+
bearerForWire: () => this.credentials.bearerForWire(),
|
|
887
|
+
presentCreds: () => this.credentials.presentCreds(),
|
|
888
|
+
presentBearer: () => this.credentials.presentBearer(),
|
|
889
|
+
armAuthExpiryReconnectFence: (nc) => this.armAuthExpiryReconnectFence(nc),
|
|
890
|
+
watchStatus: () => this.watchStatus(),
|
|
891
|
+
bindDmOnly: (mode) => this.consumers.bindDmOnly(mode),
|
|
892
|
+
requireLifecycleUid: (what) => this.requireLifecycleUid(what),
|
|
893
|
+
readPresenceOnce: () => this.readPresenceOnce(),
|
|
894
|
+
startPresenceWatch: () => this.startPresenceWatch(),
|
|
895
|
+
sweep: () => this.sweep(),
|
|
896
|
+
readChannelsOnce: () => this.channelDiscovery.readChannelsOnce(),
|
|
897
|
+
startChannelWatch: () => this.channelDiscovery.startChannelWatch(),
|
|
898
|
+
readChannels: () => this.readChannels(),
|
|
899
|
+
bindDeliveryLeaseWatch: () => this.deliveryLease.bindDeliveryLeaseWatch(),
|
|
900
|
+
ensureStreams: () => this.provisioning.ensureStreams(),
|
|
901
|
+
startConsumers: () => this.consumers.startConsumers(),
|
|
902
|
+
publishPresence: () => this.publishPresence(),
|
|
903
|
+
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
904
|
+
rearmMembershipWatches: () => this.subscriptions.rearmMembershipWatches(),
|
|
905
|
+
armPlane3: () => this.armPlane3(),
|
|
906
|
+
bindLiveness: (plane) => this.liveness.bindLiveness(plane),
|
|
907
|
+
rebindPrincipalServes: (reconnected) => this.rebindPrincipalServes(reconnected),
|
|
908
|
+
bindTap: (tap) => this.bindTap(tap),
|
|
909
|
+
retireKvWatch: (iter) => this.retireKvWatch(iter),
|
|
910
|
+
stopDeliveryLeaseWatch: () => this.deliveryLease.stopDeliveryLeaseWatch(),
|
|
911
|
+
disarmMembershipWatch: (watch) => this.disarmMembershipWatch(watch),
|
|
912
|
+
disableLibraryReconnect: (nc) => this.disableLibraryReconnect(nc),
|
|
913
|
+
dropAnycastRequests: () => this.anycastRail.dropAnycastRequests(),
|
|
914
|
+
closeWithoutLibraryReconnect: (nc) => this.credentials.closeWithoutLibraryReconnect(nc),
|
|
915
|
+
stopPrincipalServe: (intent, republish) => this.stopPrincipalServe(intent, republish),
|
|
916
|
+
kickBackoff: () => this.kickBackoff(),
|
|
917
|
+
deleteRetainedMembershipConsumer: (watch) => this.subscriptions.deleteRetainedMembershipConsumer(watch),
|
|
918
|
+
settleDeliveryLeaseWatches: () => this.settleDeliveryLeaseWatches(),
|
|
919
|
+
settleKvWatches: () => this.settleKvWatches(),
|
|
920
|
+
clearPresenceWriteFailure: () => this.presence.clearPresenceWriteFailure(),
|
|
921
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
922
|
+
});
|
|
923
|
+
/** Forwards to the connection component's {@link EndpointConnection.bindConnection}. */
|
|
924
|
+
bindConnection() {
|
|
925
|
+
return this.connection.bindConnection();
|
|
1081
926
|
}
|
|
1082
|
-
/**
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
clearConnectionScoped() {
|
|
1086
|
-
if (this.authExpiryReconnectTimer) {
|
|
1087
|
-
clearTimeout(this.authExpiryReconnectTimer);
|
|
1088
|
-
this.authExpiryReconnectTimer = undefined;
|
|
1089
|
-
}
|
|
1090
|
-
if (this.heartbeatTimer) {
|
|
1091
|
-
clearInterval(this.heartbeatTimer);
|
|
1092
|
-
this.heartbeatTimer = undefined;
|
|
1093
|
-
}
|
|
1094
|
-
if (this.sweepTimer) {
|
|
1095
|
-
clearInterval(this.sweepTimer);
|
|
1096
|
-
this.sweepTimer = undefined;
|
|
1097
|
-
}
|
|
1098
|
-
for (const msgs of this.streamMsgs) {
|
|
1099
|
-
try {
|
|
1100
|
-
msgs.stop();
|
|
1101
|
-
}
|
|
1102
|
-
catch {
|
|
1103
|
-
/* already closed with the connection */
|
|
1104
|
-
}
|
|
1105
|
-
}
|
|
1106
|
-
this.streamMsgs.length = 0;
|
|
1107
|
-
this.presenceEpoch++;
|
|
1108
|
-
this.presence.cancelRebind();
|
|
1109
|
-
this.presence.retireWatch();
|
|
1110
|
-
if (this.channelWatchIter)
|
|
1111
|
-
this.retireKvWatch(this.channelWatchIter);
|
|
1112
|
-
this.channelWatchIter = undefined;
|
|
1113
|
-
this.stopDeliveryLeaseWatch();
|
|
1114
|
-
for (const sub of this.chatSubs.values()) {
|
|
1115
|
-
try {
|
|
1116
|
-
sub.unsubscribe();
|
|
1117
|
-
}
|
|
1118
|
-
catch {
|
|
1119
|
-
/* already closed with the connection */
|
|
1120
|
-
}
|
|
1121
|
-
}
|
|
1122
|
-
this.chatSubs.clear();
|
|
1123
|
-
this.chatSubDenied.clear();
|
|
1124
|
-
this.ownWatchDeletePrefixes.clear();
|
|
1125
|
-
this.presence.resetConnection();
|
|
1126
|
-
this.channelConfigs.clear();
|
|
1127
|
-
this.channelDefaults = {};
|
|
1128
|
-
// A read in flight on the old connection must not land in the new connection's cache.
|
|
1129
|
-
this.channelRead = undefined;
|
|
1130
|
-
this.channelReadGeneration++;
|
|
1131
|
-
this.channelReadApplied = this.channelReadGeneration;
|
|
1132
|
-
for (const watch of this.membershipFeedWatches)
|
|
1133
|
-
watch.arm = watch.arm.catch(() => { }).then(() => this.disarmMembershipWatch(watch)).catch((err) => { this.emit("error", err); });
|
|
1134
|
-
}
|
|
1135
|
-
/** A transport can connect before a later JetStream/KV/subscription bind fails. The caller will
|
|
1136
|
-
* retry the whole start, so release that partial epoch first instead of overwriting `nc` and
|
|
1137
|
-
* leaving one established socket behind per retry. `close()` is deliberate: a failed bind has no
|
|
1138
|
-
* graceful delivery contract to drain, and cleanup itself must not extend the retry failure.
|
|
1139
|
-
*
|
|
1140
|
-
* A throwing `close()` is not hypothetical: nats.js `NatsConnectionImpl.close()` awaits the
|
|
1141
|
-
* protocol close, which itself flushes outbound + tears down subscription state; any of those
|
|
1142
|
-
* that reject leaves the underlying TCP transport ESTABLISHED even though every client handle
|
|
1143
|
-
* on this endpoint has been cleared. The next `start()` on the same object then dials a fresh
|
|
1144
|
-
* connection while the old socket stays live on the broker (`/varz.connections` climbs once per
|
|
1145
|
-
* failed attempt). The recovery path is layered, from most graceful to most surgical, matching
|
|
1146
|
-
* nats-core's public surface (nats.d.ts: `close`, `drain`, `isClosed`; protocol.d.ts owns the
|
|
1147
|
-
* `transport` handle whose `close(err?)` is the socket teardown; transport.d.ts exposes a
|
|
1148
|
-
* synchronous `disconnect()` for the case where even close rejects). We only touch a private
|
|
1149
|
-
* (`nc.protocol.transport`) when the public API has already refused, and the ORIGINAL bind
|
|
1150
|
-
* error is re-raised at every layer. The close error is diagnostic noise. The bind error is
|
|
1151
|
-
* what the caller must see.
|
|
1152
|
-
*
|
|
1153
|
-
* The delivery-serve subs and `this.subs` array are handles created by a successful
|
|
1154
|
-
* `armDeliveryControl` on a PREVIOUS epoch; a bind failure on the retry never rewrites them
|
|
1155
|
-
* because arming is a late step. Left in place they point at a dead protocol and survive into
|
|
1156
|
-
* the next retry. The reviewer flagged this as an untested handle class. Zero them here so a
|
|
1157
|
-
* retry starts with the same empty state as a first attempt. */
|
|
1158
|
-
async closeFailedBind() {
|
|
1159
|
-
const failedNc = this.nc;
|
|
1160
|
-
if (failedNc)
|
|
1161
|
-
this.disableLibraryReconnect(failedNc);
|
|
1162
|
-
this.clearConnectionScoped();
|
|
1163
|
-
this.nc = undefined;
|
|
1164
|
-
this.js = undefined;
|
|
1165
|
-
this.jsm = undefined;
|
|
1166
|
-
this.kv = undefined;
|
|
1167
|
-
this.channelKv = undefined;
|
|
1168
|
-
this.membersKv = undefined;
|
|
1169
|
-
this.aclKv = undefined;
|
|
1170
|
-
this.membershipFeedKv = undefined;
|
|
1171
|
-
this.deliveryLease.resetConnection();
|
|
1172
|
-
this.managerLeaseKv = undefined;
|
|
1173
|
-
this.daemonRenewalLeaseRevision = undefined;
|
|
1174
|
-
// Handles that armDeliveryControl created on a previous connection: null them so a rebind
|
|
1175
|
-
// does not carry a dead protocol's subs forward. Best-effort unsubscribe first (the sub is
|
|
1176
|
-
// dead with its connection either way; unsubscribing an already-dead sub is a noop).
|
|
1177
|
-
if (this.deliveryPlane.deliveryServeSub) {
|
|
1178
|
-
try {
|
|
1179
|
-
this.deliveryPlane.deliveryServeSub.unsubscribe();
|
|
1180
|
-
}
|
|
1181
|
-
catch { /* dead with the old connection */ }
|
|
1182
|
-
this.deliveryPlane.deliveryServeSub = undefined;
|
|
1183
|
-
}
|
|
1184
|
-
if (this.deliveryPlane.deliveryAdminServeSub) {
|
|
1185
|
-
try {
|
|
1186
|
-
this.deliveryPlane.deliveryAdminServeSub.unsubscribe();
|
|
1187
|
-
}
|
|
1188
|
-
catch { /* dead with the old connection */ }
|
|
1189
|
-
this.deliveryPlane.deliveryAdminServeSub = undefined;
|
|
1190
|
-
}
|
|
1191
|
-
this.dropAnycastRequests();
|
|
1192
|
-
for (const sub of this.subs) {
|
|
1193
|
-
try {
|
|
1194
|
-
sub.unsubscribe();
|
|
1195
|
-
}
|
|
1196
|
-
catch { /* dead with the old connection */ }
|
|
1197
|
-
}
|
|
1198
|
-
this.subs.length = 0;
|
|
1199
|
-
if (!failedNc)
|
|
1200
|
-
return;
|
|
1201
|
-
// The old status iterator is stale after nc is cleared, so its close cannot report this edge.
|
|
1202
|
-
// As in doRebuild, teardown itself must announce the no-nc window before closing the socket.
|
|
1203
|
-
this.emit("transport", { connected: false });
|
|
1204
|
-
this.emit("connection", { connected: false });
|
|
1205
|
-
// Layered teardown, comments kept OUT of the code span below on purpose: the mutation fixture
|
|
1206
|
-
// bin/smoke/mutations/failed-bind-cleanup.json anchors on that span verbatim and the fixture
|
|
1207
|
-
// census (bin/smoke/mutation-fixtures.smoke.ts) reddens an anchor that crosses a comment line.
|
|
1208
|
-
// Fast path: the public close resolved, the broker drops the connection on its own.
|
|
1209
|
-
// Layer 1: nats.js exposes drain() as the sanctioned "close on error" path. It flushes
|
|
1210
|
-
// outbound then closes, and unlike close() has its own idempotent guard against a half-closed
|
|
1211
|
-
// protocol. If drain resolves the socket is gone.
|
|
1212
|
-
// Layer 2: last resort. nats.js `NatsConnectionImpl.protocol.transport` (see
|
|
1213
|
-
// node_modules/@nats-io/nats-core/lib/{nats,protocol,transport}.d.ts) is the TCP socket the
|
|
1214
|
-
// dialer produced. Its `close(err?)` returns a Promise once the socket is torn down; its
|
|
1215
|
-
// synchronous `disconnect()` forces the FIN without waiting. Reached only after the public
|
|
1216
|
-
// API has already refused, guarded only against the shape not being what the pinned version
|
|
1217
|
-
// documents.
|
|
1218
|
-
try {
|
|
1219
|
-
await failedNc.close();
|
|
1220
|
-
return;
|
|
1221
|
-
}
|
|
1222
|
-
catch { /* fall through to the layered fallback */ }
|
|
1223
|
-
if (!failedNc.isClosed?.()) {
|
|
1224
|
-
try {
|
|
1225
|
-
await drainAndClose(failedNc);
|
|
1226
|
-
return;
|
|
1227
|
-
}
|
|
1228
|
-
catch { /* both graceful paths refused, drop to the transport */ }
|
|
1229
|
-
}
|
|
1230
|
-
const proto = failedNc.protocol;
|
|
1231
|
-
const transport = proto?.transport;
|
|
1232
|
-
if (transport && !transport.isClosed) {
|
|
1233
|
-
try {
|
|
1234
|
-
await transport.close?.();
|
|
1235
|
-
}
|
|
1236
|
-
catch { /* transport close itself threw, force it down synchronously below */ }
|
|
1237
|
-
if (!transport.isClosed) {
|
|
1238
|
-
try {
|
|
1239
|
-
transport.disconnect?.();
|
|
1240
|
-
}
|
|
1241
|
-
catch { /* nothing left to try */ }
|
|
1242
|
-
}
|
|
1243
|
-
}
|
|
927
|
+
/** Forwards to the connection component's {@link EndpointConnection.closeFailedBind}. */
|
|
928
|
+
closeFailedBind() {
|
|
929
|
+
return this.connection.closeFailedBind();
|
|
1244
930
|
}
|
|
1245
|
-
/**
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
* in doRebuild narrows to `never` via TS inlining connectAndBind's assignment). Returns
|
|
1249
|
-
* true iff it tore something down (caller bails out of the rebuild). */
|
|
1250
|
-
async tearDownIfStopped() {
|
|
1251
|
-
if (!this.stopped)
|
|
1252
|
-
return false;
|
|
1253
|
-
const nc = this.nc;
|
|
1254
|
-
if (nc)
|
|
1255
|
-
this.disableLibraryReconnect(nc);
|
|
1256
|
-
this.clearConnectionScoped();
|
|
1257
|
-
await this.closeWithoutLibraryReconnect(nc);
|
|
1258
|
-
this.nc = undefined;
|
|
1259
|
-
return true;
|
|
931
|
+
/** Forwards to the connection component's {@link EndpointConnection.tearDownIfStopped}. */
|
|
932
|
+
tearDownIfStopped() {
|
|
933
|
+
return this.connection.tearDownIfStopped();
|
|
1260
934
|
}
|
|
1261
935
|
/** Watch for a terminal close (nats.js has exhausted its own reconnect) and rebuild.
|
|
1262
936
|
* Our own stop()/drain also resolves closed(), so the `stopped` guard keeps a clean
|
|
@@ -1281,95 +955,64 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1281
955
|
void this.reestablishLoop();
|
|
1282
956
|
});
|
|
1283
957
|
}
|
|
1284
|
-
/**
|
|
1285
|
-
*
|
|
1286
|
-
|
|
1287
|
-
|
|
958
|
+
/** The rebuild and retry pacing of this endpoint. Every dependency is read when it is used, so a field
|
|
959
|
+
* assigned after construction is seen. */
|
|
960
|
+
reconnection = new EndpointReconnect({
|
|
961
|
+
rebuildPromise: () => this.rebuildPromise,
|
|
962
|
+
setRebuildPromise: (value) => { this.rebuildPromise = value; },
|
|
963
|
+
nc: () => this.nc,
|
|
964
|
+
setNc: (value) => { this.nc = value; },
|
|
965
|
+
reconnecting: () => this.reconnecting,
|
|
966
|
+
setReconnecting: (value) => { this.reconnecting = value; },
|
|
967
|
+
membershipFeedWatches: () => this.membershipFeedWatches,
|
|
968
|
+
js: () => this.js,
|
|
969
|
+
setJs: (value) => { this.js = value; },
|
|
970
|
+
jsm: () => this.jsm,
|
|
971
|
+
setJsm: (value) => { this.jsm = value; },
|
|
972
|
+
kv: () => this.kv,
|
|
973
|
+
setKv: (value) => { this.kv = value; },
|
|
974
|
+
channelKv: () => this.channelKv,
|
|
975
|
+
setChannelKv: (value) => { this.channelKv = value; },
|
|
976
|
+
membersKv: () => this.membersKv,
|
|
977
|
+
setMembersKv: (value) => { this.membersKv = value; },
|
|
978
|
+
aclKv: () => this.aclKv,
|
|
979
|
+
setAclKv: (value) => { this.aclKv = value; },
|
|
980
|
+
membershipFeedKv: () => this.membershipFeedKv,
|
|
981
|
+
setMembershipFeedKv: (value) => { this.membershipFeedKv = value; },
|
|
982
|
+
deliveryLease: () => this.deliveryLease,
|
|
983
|
+
managerLeaseKv: () => this.managerLeaseKv,
|
|
984
|
+
setManagerLeaseKv: (value) => { this.managerLeaseKv = value; },
|
|
985
|
+
daemonRenewalLeaseRevision: () => this.daemonRenewalLeaseRevision,
|
|
986
|
+
setDaemonRenewalLeaseRevision: (value) => { this.daemonRenewalLeaseRevision = value; },
|
|
987
|
+
stopped: () => this.stopped,
|
|
988
|
+
retryMs: () => this.retryMs,
|
|
989
|
+
retryAttempt: () => this.retryAttempt,
|
|
990
|
+
setRetryAttempt: (value) => { this.retryAttempt = value; },
|
|
991
|
+
backoffResolve: () => this.backoffResolve,
|
|
992
|
+
backoffTimer: () => this.backoffTimer,
|
|
993
|
+
setBackoffTimer: (value) => { this.backoffTimer = value; },
|
|
994
|
+
membershipPauses: () => this.membershipPauses,
|
|
995
|
+
disableLibraryReconnect: (nc) => this.disableLibraryReconnect(nc),
|
|
996
|
+
clearConnectionScoped: () => this.connection.clearConnectionScoped(),
|
|
997
|
+
settleDeliveryLeaseWatches: () => this.settleDeliveryLeaseWatches(),
|
|
998
|
+
settleKvWatches: () => this.settleKvWatches(),
|
|
999
|
+
closeWithoutLibraryReconnect: (nc) => this.credentials.closeWithoutLibraryReconnect(nc),
|
|
1000
|
+
connectAndBind: () => this.connectAndBind(),
|
|
1001
|
+
tearDownIfStopped: () => this.tearDownIfStopped(),
|
|
1002
|
+
superviseConnection: () => this.superviseConnection(),
|
|
1003
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
1004
|
+
});
|
|
1005
|
+
/** Forwards to the reconnect component's {@link EndpointReconnect.rebuild}. */
|
|
1288
1006
|
rebuild() {
|
|
1289
|
-
|
|
1290
|
-
return this.rebuildPromise;
|
|
1291
|
-
const p = this.doRebuild().finally(() => {
|
|
1292
|
-
if (this.rebuildPromise === p)
|
|
1293
|
-
this.rebuildPromise = undefined;
|
|
1294
|
-
});
|
|
1295
|
-
this.rebuildPromise = p;
|
|
1296
|
-
return p;
|
|
1297
|
-
}
|
|
1298
|
-
/** The transition: stop the connection-scoped timers FIRST (so nothing live touches
|
|
1299
|
-
* this.nc during the null window), drop the connection refs, close the old nc, then
|
|
1300
|
-
* rebind + re-arm the supervisor on the fresh connection. clearConnectionScoped is
|
|
1301
|
-
* idempotent, so connectAndBind's own call here is a noop. */
|
|
1302
|
-
async doRebuild() {
|
|
1303
|
-
const oldNc = this.nc;
|
|
1304
|
-
if (oldNc)
|
|
1305
|
-
this.disableLibraryReconnect(oldNc);
|
|
1306
|
-
this.reconnecting = true;
|
|
1307
|
-
try {
|
|
1308
|
-
this.clearConnectionScoped();
|
|
1309
|
-
// Manual reconnect still has a live old epoch: complete broker-consumer cleanup before close.
|
|
1310
|
-
// Terminal self-heal has an already-closed epoch: disarm retains stream/name for fresh cleanup,
|
|
1311
|
-
// and the KV watch consumers that closed with it are left to the broker's inactive threshold.
|
|
1312
|
-
if (oldNc && !oldNc.isClosed())
|
|
1313
|
-
await Promise.all([...[...this.membershipFeedWatches].map((watch) => watch.arm), this.settleDeliveryLeaseWatches(), this.settleKvWatches()]);
|
|
1314
|
-
this.nc = undefined;
|
|
1315
|
-
this.js = undefined;
|
|
1316
|
-
this.jsm = undefined;
|
|
1317
|
-
this.kv = undefined;
|
|
1318
|
-
this.channelKv = undefined;
|
|
1319
|
-
// Plane-3 KV handles are bound to the old connection too — drop them so the daemon re-opens them on
|
|
1320
|
-
// the fresh nc (else durableJoin/leave/list, the reader's ACL re-auth, and lease renew use a dead
|
|
1321
|
-
// handle after a reconnect).
|
|
1322
|
-
this.membersKv = undefined;
|
|
1323
|
-
this.aclKv = undefined;
|
|
1324
|
-
this.membershipFeedKv = undefined;
|
|
1325
|
-
this.deliveryLease.resetConnection();
|
|
1326
|
-
// The manager's liveness-lease handle too: left bound to the old connection, every renew and
|
|
1327
|
-
// re-read after a reconnect times out, and the manager reports its lease unknown for good.
|
|
1328
|
-
this.managerLeaseKv = undefined;
|
|
1329
|
-
this.daemonRenewalLeaseRevision = undefined;
|
|
1330
|
-
// This is an application-requested epoch teardown, not a transient nats.js blip. The old
|
|
1331
|
-
// status iterator is now stale by construction and its close is epoch-dropped, so this line is
|
|
1332
|
-
// the authoritative raw-liveness edge for the no-nc window until the new watcher seeds true.
|
|
1333
|
-
this.emit("transport", { connected: false });
|
|
1334
|
-
this.emit("connection", { connected: false });
|
|
1335
|
-
await this.closeWithoutLibraryReconnect(oldNc);
|
|
1336
|
-
// stop() can arrive while the old epoch closes. Do not start another authentication
|
|
1337
|
-
// handshake after it: a terminal credential may already have expired in that window.
|
|
1338
|
-
// Keep old-epoch and late-epoch close failures outside this dial-only catch.
|
|
1339
|
-
if (this.stopped)
|
|
1340
|
-
return;
|
|
1341
|
-
try {
|
|
1342
|
-
await this.connectAndBind();
|
|
1343
|
-
}
|
|
1344
|
-
catch (error) {
|
|
1345
|
-
if (!this.stopped)
|
|
1346
|
-
throw error;
|
|
1347
|
-
}
|
|
1348
|
-
// stop() may have run during the await — don't leave a live connection + heartbeat +
|
|
1349
|
-
// supervisor on a stopped endpoint. (Reads this.nc in its own scope — a bare `this.nc`
|
|
1350
|
-
// here in doRebuild narrows to `never` via TS inlining connectAndBind's assignment.)
|
|
1351
|
-
// Re-arm on the fresh nc only after this stopped fence accepts it.
|
|
1352
|
-
if (await this.tearDownIfStopped())
|
|
1353
|
-
return;
|
|
1354
|
-
this.superviseConnection();
|
|
1355
|
-
}
|
|
1356
|
-
finally {
|
|
1357
|
-
this.reconnecting = false;
|
|
1358
|
-
}
|
|
1007
|
+
return this.reconnection.rebuild();
|
|
1359
1008
|
}
|
|
1360
|
-
/**
|
|
1361
|
-
* a standing one (a down broker, a credential nothing can renew), and retrying it every few
|
|
1362
|
-
* seconds forever is what turns one stuck client into a permanent flat load on the broker and its
|
|
1363
|
-
* auth callout. */
|
|
1364
|
-
static RETRY_BACKOFF_CAP_MS = 60_000;
|
|
1365
|
-
/** The wait before the next rebuild attempt: {@link retryMs}, doubling per consecutive failure up
|
|
1366
|
-
* to {@link RETRY_BACKOFF_CAP_MS}, then a random half of it taken off. A transient drop still
|
|
1367
|
-
* recovers within retryMs, and endpoints refused together, as a broker restart or an auth limit
|
|
1368
|
-
* refuses a host's agents, spread their retries instead of arriving at the callout at once. */
|
|
1009
|
+
/** Forwards to the reconnect component's {@link EndpointReconnect.nextRetryDelayMs}. */
|
|
1369
1010
|
nextRetryDelayMs() {
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1011
|
+
return this.reconnection.nextRetryDelayMs();
|
|
1012
|
+
}
|
|
1013
|
+
/** Forwards to the reconnect component's {@link EndpointReconnect.kickBackoff}. */
|
|
1014
|
+
kickBackoff() {
|
|
1015
|
+
return this.reconnection.kickBackoff();
|
|
1373
1016
|
}
|
|
1374
1017
|
/** Rebuild with backoff until it sticks or we're stopped. Interruptible: a manual
|
|
1375
1018
|
* {@link reconnect} kicks the backoff so the next attempt runs immediately instead of
|
|
@@ -1419,30 +1062,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1419
1062
|
if (!this.stopped && this.reestablishing && !this.rebuildPromise)
|
|
1420
1063
|
this.kickBackoff();
|
|
1421
1064
|
}
|
|
1422
|
-
/** Cut an in-flight reestablish backoff short so the next attempt runs immediately, and
|
|
1423
|
-
* clear its timer so it can't fire later on a stopped/restarted loop. */
|
|
1424
|
-
kickBackoff() {
|
|
1425
|
-
this.backoffResolve?.();
|
|
1426
|
-
if (this.backoffTimer) {
|
|
1427
|
-
clearTimeout(this.backoffTimer);
|
|
1428
|
-
this.backoffTimer = undefined;
|
|
1429
|
-
}
|
|
1430
|
-
}
|
|
1431
|
-
/** Capped exponential pause between membership retries. stop() ends it early, so a loop waiting
|
|
1432
|
-
* on an absent delivery daemon never holds a stopped endpoint's process open for the delay. */
|
|
1433
|
-
membershipPause(attempt) {
|
|
1434
|
-
if (this.stopped)
|
|
1435
|
-
return Promise.resolve();
|
|
1436
|
-
return new Promise((resolve) => {
|
|
1437
|
-
const wake = () => {
|
|
1438
|
-
clearTimeout(timer);
|
|
1439
|
-
this.membershipPauses.delete(wake);
|
|
1440
|
-
resolve();
|
|
1441
|
-
};
|
|
1442
|
-
const timer = setTimeout(wake, Math.min(30_000, 1000 * 2 ** attempt));
|
|
1443
|
-
this.membershipPauses.add(wake);
|
|
1444
|
-
});
|
|
1445
|
-
}
|
|
1446
1065
|
/** Manual reconnect: tear down the current connection and rebuild, WITHOUT the permanent
|
|
1447
1066
|
* stop (stopped/stopping stay false). Serialized with the self-heal supervisor via
|
|
1448
1067
|
* {@link rebuild}, and interruptible — if a backoff is in flight, kick it so the attempt
|
|
@@ -1474,99 +1093,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1474
1093
|
stop() {
|
|
1475
1094
|
return this.stopPromise ??= this.stopOnce();
|
|
1476
1095
|
}
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
this.stopped = true;
|
|
1481
|
-
for (const follower of this.activeGoalFollowers) {
|
|
1482
|
-
follower.cancel();
|
|
1483
|
-
}
|
|
1484
|
-
this.activeGoalFollowers.clear();
|
|
1485
|
-
this.presenceEpoch++;
|
|
1486
|
-
// Every principal serve ends now, so a `ready` still waiting for the broker rejects and no new
|
|
1487
|
-
// request is taken. Its handlers in flight finish below, before the connection they reply on
|
|
1488
|
-
// closes, or their requests' deadlines pass.
|
|
1489
|
-
const principalStops = [...this.principalServes.values()].map((intent) => this.stopPrincipalServe(intent, false));
|
|
1490
|
-
this.presence.cancelRebind();
|
|
1491
|
-
// Wake a reestablishLoop sitting in backoff so it sees `stopped` and exits instead of
|
|
1492
|
-
// sleeping out retryMs; also clears the timer so it can't fire later.
|
|
1493
|
-
this.kickBackoff();
|
|
1494
|
-
for (const wake of [...this.membershipPauses])
|
|
1495
|
-
wake();
|
|
1496
|
-
if (this.heartbeatTimer)
|
|
1497
|
-
clearInterval(this.heartbeatTimer);
|
|
1498
|
-
if (this.sweepTimer)
|
|
1499
|
-
clearInterval(this.sweepTimer);
|
|
1500
|
-
this.cancelBearerRenewal?.();
|
|
1501
|
-
if (this.credsTimer)
|
|
1502
|
-
clearTimeout(this.credsTimer);
|
|
1503
|
-
if (this.authExpiryReconnectTimer)
|
|
1504
|
-
clearTimeout(this.authExpiryReconnectTimer);
|
|
1505
|
-
for (const watch of this.membershipFeedWatches) {
|
|
1506
|
-
watch.stopped = true;
|
|
1507
|
-
watch.arm = watch.arm.catch(() => { }).then(async () => {
|
|
1508
|
-
await this.disarmMembershipWatch(watch);
|
|
1509
|
-
await this.deleteRetainedMembershipConsumer(watch);
|
|
1510
|
-
});
|
|
1511
|
-
}
|
|
1512
|
-
// Permanent endpoint shutdown has no future epoch. Try strict cleanup on the current live epoch;
|
|
1513
|
-
// if the broker is already gone, terminate local ownership rather than hanging shutdown forever.
|
|
1514
|
-
await Promise.all([...this.membershipFeedWatches].map((watch) => watch.arm.catch((err) => {
|
|
1515
|
-
if (this.nc && !this.nc.isClosed())
|
|
1516
|
-
throw err;
|
|
1517
|
-
})));
|
|
1518
|
-
for (const watch of this.membershipFeedWatches) {
|
|
1519
|
-
watch.resolveStop?.();
|
|
1520
|
-
watch.resolveStop = undefined;
|
|
1521
|
-
watch.rejectStop = undefined;
|
|
1522
|
-
}
|
|
1523
|
-
this.membershipFeedWatches.clear();
|
|
1524
|
-
for (const msgs of this.streamMsgs) {
|
|
1525
|
-
try {
|
|
1526
|
-
msgs.stop();
|
|
1527
|
-
}
|
|
1528
|
-
catch {
|
|
1529
|
-
/* already closed */
|
|
1530
|
-
}
|
|
1531
|
-
}
|
|
1532
|
-
this.presence.retireWatch();
|
|
1533
|
-
if (this.channelWatchIter)
|
|
1534
|
-
this.retireKvWatch(this.channelWatchIter);
|
|
1535
|
-
this.channelWatchIter = undefined;
|
|
1536
|
-
this.stopDeliveryLeaseWatch();
|
|
1537
|
-
await Promise.all(principalStops);
|
|
1538
|
-
try {
|
|
1539
|
-
if (this.doRegister) {
|
|
1540
|
-
await this.presence.publishOffline();
|
|
1541
|
-
}
|
|
1542
|
-
}
|
|
1543
|
-
catch {
|
|
1544
|
-
/* best-effort graceful leave */
|
|
1545
|
-
}
|
|
1546
|
-
await Promise.all([this.settleDeliveryLeaseWatches(), this.settleKvWatches()]);
|
|
1547
|
-
// DM-only: the pull above is stopped, so no new copy reaches the handler. Let a running handler
|
|
1548
|
-
// settle while the connection is still open, so its ack can reach the broker; close either way.
|
|
1549
|
-
let dmOnlyUnsettled = false;
|
|
1550
|
-
if (this.dmOnly && this.dmOnlyInFlight) {
|
|
1551
|
-
const settleMs = this.dmOnly.settleTimeoutMs ?? 30_000;
|
|
1552
|
-
dmOnlyUnsettled = await withDeadline(this.dmOnlyInFlight.then(() => false), settleMs, () => new Error("unsettled"))
|
|
1553
|
-
.catch(() => true);
|
|
1554
|
-
}
|
|
1555
|
-
await this.closeWithoutLibraryReconnect(this.nc);
|
|
1556
|
-
// #1356: stop() does its own teardown and never runs clearConnectionScoped, so clear here too —
|
|
1557
|
-
// AFTER the best-effort offline publish above, which would otherwise re-record a refusal on its
|
|
1558
|
-
// way out. A stopped endpoint has no live basis for "this bucket is refusing writes", and the
|
|
1559
|
-
// duration is computed from `Date.now()`, so a retained record does not merely go stale: it keeps
|
|
1560
|
-
// COUNTING UP for as long as the dead object is held.
|
|
1561
|
-
this.clearPresenceWriteFailure();
|
|
1562
|
-
// Custody callers must not observe stop completion while a concurrent rebind can still
|
|
1563
|
-
// publish a late consuming connection. doRebuild closes its stopped epoch before resolving.
|
|
1564
|
-
await this.rebuildPromise;
|
|
1565
|
-
await this.startPromise;
|
|
1566
|
-
if ([...this.ownedConnections].some((nc) => !nc.isClosed()))
|
|
1567
|
-
throw new Error("endpoint stop could not prove its connection closed; local consumer custody must be retained");
|
|
1568
|
-
if (dmOnlyUnsettled)
|
|
1569
|
-
throw new Error(`endpoint stop closed the connection but the DM-only handler did not settle within ${this.dmOnly?.settleTimeoutMs ?? 30_000}ms; its copy stays unacked and handler custody must be retained`);
|
|
1096
|
+
/** Forwards to the connection component's {@link EndpointConnection.stopOnce}. */
|
|
1097
|
+
stopOnce() {
|
|
1098
|
+
return this.connection.stopOnce();
|
|
1570
1099
|
}
|
|
1571
1100
|
/** The native binding of every connection this endpoint holds. A host compares the list with a
|
|
1572
1101
|
* complete native census of the account to show that no other connection uses it. The list
|
|
@@ -1575,20 +1104,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1575
1104
|
* start, during a start, rebuild or stop, while a credential transaction or its connection swap
|
|
1576
1105
|
* is pending, and while the connection is down or redialing. */
|
|
1577
1106
|
nativeConnections() {
|
|
1578
|
-
|
|
1579
|
-
if (this.startPromise !== undefined || this.rebuildPromise !== undefined || this.reconnecting)
|
|
1580
|
-
throw unsettled("a start or rebuild is in flight");
|
|
1581
|
-
const open = [...this.ownedConnections].filter((nc) => !nc.isClosed());
|
|
1582
|
-
if (this.stopped) {
|
|
1583
|
-
if (open.length > 0)
|
|
1584
|
-
throw unsettled("the endpoint stopped but a connection has not closed");
|
|
1585
|
-
return [];
|
|
1586
|
-
}
|
|
1587
|
-
if (this.credsTxnsRunning > 0 || this.residentSwap !== undefined)
|
|
1588
|
-
throw unsettled("a credential adoption is in flight");
|
|
1589
|
-
if (!this.nc || !this.nativeBound)
|
|
1590
|
-
throw unsettled("the connection is down or redialing");
|
|
1591
|
-
return open.map((nc) => nativeConnectionBinding(nc, "endpoint connection"));
|
|
1107
|
+
return this.connection.nativeConnections();
|
|
1592
1108
|
}
|
|
1593
1109
|
// ---- messaging -----------------------------------------------------------
|
|
1594
1110
|
/** The messaging verbs of this endpoint: multicast, unicast, anycast and the event-payload
|
|
@@ -2125,9 +1641,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2125
1641
|
readChannels: () => this.readChannels(),
|
|
2126
1642
|
armJoin: (channels) => this.armJoin(channels),
|
|
2127
1643
|
backfillArmed: (frontiers, floorSeq) => this.backfillArmed(frontiers, floorSeq),
|
|
2128
|
-
subscribeChat: (channel) => this.subscribeChat(channel),
|
|
2129
|
-
unsubscribeChat: (channel) => this.unsubscribeChat(channel),
|
|
2130
|
-
confirmChatSub: () => this.confirmChatSub(),
|
|
1644
|
+
subscribeChat: (channel) => this.consumers.subscribeChat(channel),
|
|
1645
|
+
unsubscribeChat: (channel) => this.consumers.unsubscribeChat(channel),
|
|
1646
|
+
confirmChatSub: () => this.consumers.confirmChatSub(),
|
|
2131
1647
|
durableJoinChannel: (channel) => this.durableJoinChannel(channel),
|
|
2132
1648
|
durableLeaveChannel: (...args) => this.durableLeaveChannel(...args),
|
|
2133
1649
|
fetchMemberships: (channel) => this.fetchMemberships(channel),
|
|
@@ -2240,10 +1756,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2240
1756
|
async watchMembership(onChange, onClosed) {
|
|
2241
1757
|
return this.subscriptions.watchMembership(onChange, onClosed);
|
|
2242
1758
|
}
|
|
2243
|
-
/** Forwards to the subscriptions component's {@link EndpointSubscriptions.deleteRetainedMembershipConsumer}. */
|
|
2244
|
-
async deleteRetainedMembershipConsumer(watch) {
|
|
2245
|
-
return this.subscriptions.deleteRetainedMembershipConsumer(watch);
|
|
2246
|
-
}
|
|
2247
1759
|
/** Record the consumer behind one of this connection's own KV watches or membership scans, and
|
|
2248
1760
|
* return the delete-subject prefix it recorded. nats.js names it `<prefix>_<serial>` and,
|
|
2249
1761
|
* rebuilding it after a stall or a sequence gap, deletes the predecessor itself with no hook before
|
|
@@ -2300,10 +1812,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2300
1812
|
async disarmMembershipWatch(watch) {
|
|
2301
1813
|
return this.subscriptions.disarmMembershipWatch(watch);
|
|
2302
1814
|
}
|
|
2303
|
-
/** Forwards to the subscriptions component's {@link EndpointSubscriptions.rearmMembershipWatches}. */
|
|
2304
|
-
async rearmMembershipWatches() {
|
|
2305
|
-
return this.subscriptions.rearmMembershipWatches();
|
|
2306
|
-
}
|
|
2307
1815
|
/** Fetch recent messages from a channel's JetStream backlog. `signal` cancels the active pull and
|
|
2308
1816
|
* reclaims its ephemeral consumer before the promise rejects. Returns `HistoryMessage[]`
|
|
2309
1817
|
* (#1413): every field present on a row was checked by the drain before the row was
|
|
@@ -2463,7 +1971,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2463
1971
|
* failing call — so without this an over-tight ACL silently drops the agent's
|
|
2464
1972
|
* traffic and it just looks "absent". A denied request is the exception: nats.js also
|
|
2465
1973
|
* rejects that request with the denial, so its caller owns it (see {@link trackRequestDenials}).
|
|
2466
|
-
* So is a refused channel subscription, whose callback owns it (see {@link subscribeChat}).
|
|
1974
|
+
* So is a refused channel subscription, whose callback owns it (see {@link EndpointConsumers.subscribeChat}).
|
|
2467
1975
|
* We annotate permission denials explicitly so a denial is never mistaken for absence
|
|
2468
1976
|
* (which already has a benign cause: MCP reconnect).
|
|
2469
1977
|
*/
|
|
@@ -2625,10 +2133,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2625
2133
|
notLiveMsg: () => this.notLiveMsg(),
|
|
2626
2134
|
manager: () => this.manager(),
|
|
2627
2135
|
});
|
|
2628
|
-
/** Forwards to the provisioning component's {@link EndpointProvisioning.ensureStreams}. */
|
|
2629
|
-
async ensureStreams() {
|
|
2630
|
-
return this.provisioning.ensureStreams();
|
|
2631
|
-
}
|
|
2632
2136
|
/** Create this open space's message streams if they are not there yet, as a consuming endpoint
|
|
2633
2137
|
* does when it starts. For a sender that consumes nothing and may be the space's first
|
|
2634
2138
|
* participant (#7045). An authenticated broker's streams come from its setup, and its agents may
|
|
@@ -2646,10 +2150,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2646
2150
|
async provisionDmInbox(owner, actor, lifecycleUid) {
|
|
2647
2151
|
return this.provisioning.provisionDmInbox(owner, actor, lifecycleUid);
|
|
2648
2152
|
}
|
|
2649
|
-
/** Forwards to the provisioning component's {@link EndpointProvisioning.ensureDmDurable}. */
|
|
2650
|
-
async ensureDmDurable(owner, actor, lifecycleUid, opts) {
|
|
2651
|
-
return this.provisioning.ensureDmDurable(owner, actor, lifecycleUid, opts);
|
|
2652
|
-
}
|
|
2653
2153
|
/**
|
|
2654
2154
|
* Privileged: pre-create an agent's bind-only Plane-3 DELIVER durable (`dlv_<id>`, filtered to
|
|
2655
2155
|
* `dlv.<id>`), so the agent can BIND its per-member durable handoff without holding CONSUMER.CREATE
|
|
@@ -2700,11 +2200,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2700
2200
|
daemonRenewalLeaseRevision: () => this.daemonRenewalLeaseRevision,
|
|
2701
2201
|
setDaemonRenewalLeaseRevision: (revision) => { this.daemonRenewalLeaseRevision = revision; },
|
|
2702
2202
|
notLiveMsg: () => this.notLiveMsg(),
|
|
2703
|
-
consumerInfo: (stream, durable) => this.consumerInfo(stream, durable),
|
|
2203
|
+
consumerInfo: (stream, durable) => this.consumers.consumerInfo(stream, durable),
|
|
2704
2204
|
manager: () => this.manager(),
|
|
2705
2205
|
chatFrontier: () => this.chatFrontier(),
|
|
2706
2206
|
aclAllows: (owner, lifecycleUid, channel) => this.aclAllows(owner, lifecycleUid, channel),
|
|
2707
|
-
catchupCopy: (...args) => this.catchupCopy(...args),
|
|
2207
|
+
catchupCopy: (...args) => this.durable.catchupCopy(...args),
|
|
2708
2208
|
});
|
|
2709
2209
|
/** Forwards to the durable backstop's {@link EndpointDurableBackstop.membersRegistry}. */
|
|
2710
2210
|
async membersRegistry() {
|
|
@@ -2840,9 +2340,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2840
2340
|
async watchDeliveryLease(shardIndex, onEvent) {
|
|
2841
2341
|
return this.deliveryLease.watchDeliveryLease(shardIndex, onEvent);
|
|
2842
2342
|
}
|
|
2843
|
-
stopDeliveryLeaseWatch() {
|
|
2844
|
-
return this.deliveryLease.stopDeliveryLeaseWatch();
|
|
2845
|
-
}
|
|
2846
2343
|
/** Wait for the lease-watch binds still in flight, then for every retired watch's consumer delete.
|
|
2847
2344
|
* {@link stop} and a live-epoch rebuild run this before they close the connection those deletes
|
|
2848
2345
|
* need (#3966). */
|
|
@@ -2898,12 +2395,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2898
2395
|
void binding.then(settled, settled);
|
|
2899
2396
|
return binding;
|
|
2900
2397
|
}
|
|
2901
|
-
/** Bind the lease watch on the current connection. A bind that lands after a newer watch replaced
|
|
2902
|
-
* its intent, or after a stop or connection teardown moved the epoch, retires its watch instead of
|
|
2903
|
-
* installing it: the epoch it was bound for is gone, and a rebuilt one binds its own. */
|
|
2904
|
-
bindDeliveryLeaseWatch() {
|
|
2905
|
-
return this.deliveryLease.bindDeliveryLeaseWatch();
|
|
2906
|
-
}
|
|
2907
2398
|
/** The principal recorded as holding a shard's delivery lease, read by THIS process from the row,
|
|
2908
2399
|
* or `undefined` when no row is live or the read failed.
|
|
2909
2400
|
*
|
|
@@ -3025,18 +2516,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3025
2516
|
async deliveryClassFresh(channel) {
|
|
3026
2517
|
return this.durableBackstop.deliveryClassFresh(channel);
|
|
3027
2518
|
}
|
|
3028
|
-
/** Forwards to the durable backstop's {@link EndpointDurableBackstop.resolveOwnerByName}. */
|
|
3029
|
-
resolveOwnerByName(name) {
|
|
3030
|
-
return this.durableBackstop.resolveOwnerByName(name);
|
|
3031
|
-
}
|
|
3032
|
-
/** Forwards to the durable backstop's {@link EndpointDurableBackstop.plane3MsgId}. */
|
|
3033
|
-
plane3MsgId(entry, keySuffix) {
|
|
3034
|
-
return this.durableBackstop.plane3MsgId(entry, keySuffix);
|
|
3035
|
-
}
|
|
3036
|
-
/** Forwards to the durable backstop's {@link EndpointDurableBackstop.publishDinbox}. */
|
|
3037
|
-
async publishDinbox(principal, lifecycleUid, entry) {
|
|
3038
|
-
return this.durableBackstop.publishDinbox(principal, lifecycleUid, entry);
|
|
3039
|
-
}
|
|
3040
2519
|
/**
|
|
3041
2520
|
* Privileged durable-JOIN write (v3: the delivery daemon calls this from its `ctl.delivery` handler
|
|
3042
2521
|
* after validating channel ⊆ the caller's read ACL): capture `joinCursor`, commit a `durable-active`
|
|
@@ -3061,95 +2540,33 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3061
2540
|
async durableJoinFor(owner, channel, lifecycleUid, opts = {}) {
|
|
3062
2541
|
return this.durableBackstop.durableJoinFor(owner, channel, lifecycleUid, opts);
|
|
3063
2542
|
}
|
|
3064
|
-
/**
|
|
3065
|
-
*
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
2543
|
+
/** The agent-side durable membership of this endpoint. Every dependency is read when it is used, so a
|
|
2544
|
+
* field assigned after construction is seen. */
|
|
2545
|
+
durable = new EndpointDurable({
|
|
2546
|
+
js: () => this.js,
|
|
2547
|
+
jsm: () => this.jsm,
|
|
2548
|
+
space: () => this.space,
|
|
2549
|
+
pendingDurableLeave: () => this.pendingDurableLeave,
|
|
2550
|
+
stopped: () => this.stopped,
|
|
2551
|
+
plane3Channels: () => this.plane3Channels,
|
|
2552
|
+
aclRegistry: () => this.aclRegistry(),
|
|
2553
|
+
deliveryClassFresh: (channel) => this.deliveryClassFresh(channel),
|
|
2554
|
+
deleteConsumerIfPresent: (stream, durable) => this.consumers.deleteConsumerIfPresent(stream, durable),
|
|
2555
|
+
publishDinbox: (principal, lifecycleUid, entry) => this.durableBackstop.publishDinbox(principal, lifecycleUid, entry),
|
|
2556
|
+
requestDelivery: (op, args, timeoutMs) => this.requestDelivery(op, args, timeoutMs),
|
|
2557
|
+
requireLifecycleUid: (what) => this.requireLifecycleUid(what),
|
|
2558
|
+
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
2559
|
+
membershipPause: (attempt) => this.reconnection.membershipPause(attempt),
|
|
2560
|
+
});
|
|
2561
|
+
/** Forwards to the durable-membership component's {@link EndpointDurable.aclAllows}. */
|
|
2562
|
+
aclAllows(owner, lifecycleUid, channel) {
|
|
2563
|
+
return this.durable.aclAllows(owner, lifecycleUid, channel);
|
|
3070
2564
|
}
|
|
3071
2565
|
/** Privileged durable-LEAVE write: tombstone the membership at `leaveCursor = frontier` so the
|
|
3072
2566
|
* backstop denies `seq > leaveCursor` while a pre-leave entry stays deliverable (SPEC §7 interval). */
|
|
3073
2567
|
async durableLeaveFor(owner, channel, lifecycleUid, expectedGeneration) {
|
|
3074
2568
|
return this.durableBackstop.durableLeaveFor(owner, channel, lifecycleUid, expectedGeneration);
|
|
3075
2569
|
}
|
|
3076
|
-
/** Idempotently copy the eligible chat messages in `(fromSeqExcl, toSeqIncl]` for `channel` into the
|
|
3077
|
-
* owner inbox, via a DEDICATED per-(owner,join) ephemeral consumer (NOT the agent-scoped
|
|
3078
|
-
* `chathist_<id>`/`histLock` — red-team HIGH-8). It copies what the stream still holds in that
|
|
3079
|
-
* window; a post the history limit removed first is not copied and is not reported (SPEC §4).
|
|
3080
|
-
*
|
|
3081
|
-
* A FAMILY row (`review.>`, #5404) reads the window on every channel under it and copies a post only
|
|
3082
|
-
* when that post's own channel is durable-class. Each copy names the post's own channel, never the
|
|
3083
|
-
* family, and it skips nothing another row of the lifecycle covers: two joins can both be pending
|
|
3084
|
-
* when a post lands, so neither proves the other copied it. The per-message, per-lifecycle dedupe id
|
|
3085
|
-
* keeps one copy. */
|
|
3086
|
-
async catchupCopy(owner, lifecycleUid, channel, fromSeqExcl, toSeqIncl, generation) {
|
|
3087
|
-
if (!this.js || !this.jsm || toSeqIncl <= fromSeqExcl)
|
|
3088
|
-
return 0;
|
|
3089
|
-
const subject = chatSubject(this.space, "*", "*", channel);
|
|
3090
|
-
const family = isChannelFamily(channel);
|
|
3091
|
-
const durableClass = new Map();
|
|
3092
|
-
const stored = (ch) => {
|
|
3093
|
-
if (!family)
|
|
3094
|
-
return Promise.resolve(true);
|
|
3095
|
-
let cls = durableClass.get(ch);
|
|
3096
|
-
if (cls === undefined)
|
|
3097
|
-
durableClass.set(ch, (cls = this.deliveryClassFresh(ch).then((c) => c === "durable")));
|
|
3098
|
-
return cls;
|
|
3099
|
-
};
|
|
3100
|
-
// Consumer NAME must be JetStream-safe (no `.`) AND collision-free: use the principal DASH-form
|
|
3101
|
-
// (`<owner>-<actor>`, `-` reserved as the sole separator), NOT `token(owner)` — `token()` maps the
|
|
3102
|
-
// dot-form `.`→`_`, which is NOT collision-free (`_` is legal inside a token, so `a.b_c` and `a_b.c`
|
|
3103
|
-
// would both underscore to `a_b_c`). LIFECYCLE-KEYED (SPEC 13.1): generations restart at 1 per
|
|
3104
|
-
// lifecycle, so an alias-keyed `cu_<principal>_<gen>` would let a same-alias successor
|
|
3105
|
-
// delete/recreate a predecessor's in-flight catch-up consumer — the uid disambiguates.
|
|
3106
|
-
const cuP = parsePrincipalKey(owner);
|
|
3107
|
-
const name = `cu_${cuP ? lifecycleNameKey(cuP.owner, cuP.actor, lifecycleUid) : `${token(owner)}-${lifecycleUid}`}_${generation}`;
|
|
3108
|
-
// A failed read throws, leaving the pending row to retry over the same window.
|
|
3109
|
-
await this.deleteConsumerIfPresent(chatStream(this.space), name);
|
|
3110
|
-
await this.jsm.consumers.add(chatStream(this.space), {
|
|
3111
|
-
name, filter_subject: subject, ack_policy: AckPolicy.None, mem_storage: true,
|
|
3112
|
-
inactive_threshold: nanos(30_000), deliver_policy: DeliverPolicy.StartSequence, opt_start_seq: fromSeqExcl + 1,
|
|
3113
|
-
});
|
|
3114
|
-
let copied = 0;
|
|
3115
|
-
try {
|
|
3116
|
-
const consumer = await this.js.consumers.get(chatStream(this.space), name);
|
|
3117
|
-
let pending = (await consumer.info()).num_pending;
|
|
3118
|
-
while (pending > 0) {
|
|
3119
|
-
const want = Math.min(pending, 256);
|
|
3120
|
-
const iter = await consumer.fetch({ max_messages: want, expires: 5_000 });
|
|
3121
|
-
let got = 0;
|
|
3122
|
-
for await (const m of iter) {
|
|
3123
|
-
got++;
|
|
3124
|
-
if (m.seq > toSeqIncl)
|
|
3125
|
-
return copied;
|
|
3126
|
-
let msg;
|
|
3127
|
-
try {
|
|
3128
|
-
msg = m.json();
|
|
3129
|
-
}
|
|
3130
|
-
catch {
|
|
3131
|
-
continue;
|
|
3132
|
-
}
|
|
3133
|
-
const parsed = parseSubject(m.subject);
|
|
3134
|
-
if (!parsed || !isFromSubjectSender(msg, parsed) || msg.from.id === owner)
|
|
3135
|
-
continue;
|
|
3136
|
-
if (!(await stored(parsed.rest)))
|
|
3137
|
-
continue;
|
|
3138
|
-
await this.publishDinbox(owner, lifecycleUid, {
|
|
3139
|
-
msg, channel: parsed.rest, seq: m.seq, reason: "durable-channel", generation, ...(family ? { row: channel } : {}),
|
|
3140
|
-
});
|
|
3141
|
-
copied++;
|
|
3142
|
-
}
|
|
3143
|
-
if (got < want)
|
|
3144
|
-
break;
|
|
3145
|
-
pending -= got;
|
|
3146
|
-
}
|
|
3147
|
-
}
|
|
3148
|
-
finally {
|
|
3149
|
-
await this.deleteConsumerIfPresent(chatStream(this.space), name);
|
|
3150
|
-
}
|
|
3151
|
-
return copied;
|
|
3152
|
-
}
|
|
3153
2570
|
/** Start the Plane-3 fan-out writer + trusted reader on THIS (privileged, server-side delivery-daemon)
|
|
3154
2571
|
* endpoint, AND serve the `ctl.delivery` control service (runtime durable join/leave/list). `aclFor`
|
|
3155
2572
|
* maps an owner id to its current read ACL for the reader's re-authorization — read FRESH per entry
|
|
@@ -3201,195 +2618,40 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3201
2618
|
async rearmPlane3() {
|
|
3202
2619
|
return this.deliveryPlane.rearmPlane3();
|
|
3203
2620
|
}
|
|
3204
|
-
/** Forwards to the delivery plane's {@link EndpointDeliveryPlane.plane3MayAct}. */
|
|
3205
|
-
plane3MayAct() {
|
|
3206
|
-
return this.deliveryPlane.plane3MayAct();
|
|
3207
|
-
}
|
|
3208
2621
|
/** Forwards to the delivery plane's {@link EndpointDeliveryPlane.armPlane3}. */
|
|
3209
2622
|
async armPlane3() {
|
|
3210
2623
|
return this.deliveryPlane.armPlane3();
|
|
3211
2624
|
}
|
|
3212
|
-
/**
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
this.
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
}
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
this.
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
send: (to, event) => this.sendAnycastNotice(to, event),
|
|
3243
|
-
mayAct: () => this.plane3MayAct(),
|
|
3244
|
-
onError: (e) => this.emitRecoverable(e),
|
|
3245
|
-
});
|
|
3246
|
-
this.anycastAuthority = authority;
|
|
3247
|
-
this.takeServeSub = this.serveTakes(authority);
|
|
3248
|
-
const sweep = () => void authority.sweep().catch((e) => this.emitRecoverable(e));
|
|
3249
|
-
this.anycastSweep = setInterval(sweep, ANYCAST_SWEEP_MS);
|
|
3250
|
-
this.anycastSweep.unref?.();
|
|
3251
|
-
sweep();
|
|
3252
|
-
}
|
|
3253
|
-
dropAnycastRequests() {
|
|
3254
|
-
if (this.takeServeSub) {
|
|
3255
|
-
try {
|
|
3256
|
-
this.takeServeSub.unsubscribe();
|
|
3257
|
-
}
|
|
3258
|
-
catch { /* dead with the old connection */ }
|
|
3259
|
-
const i = this.subs.indexOf(this.takeServeSub);
|
|
3260
|
-
if (i >= 0)
|
|
3261
|
-
this.subs.splice(i, 1);
|
|
3262
|
-
this.takeServeSub = undefined;
|
|
3263
|
-
}
|
|
3264
|
-
clearInterval(this.anycastSweep);
|
|
3265
|
-
this.anycastSweep = undefined;
|
|
3266
|
-
this.anycastAuthority = undefined;
|
|
3267
|
-
}
|
|
3268
|
-
/** Serve takes on `take.*.*.*`, each in its own time: a take reads one request and writes one row,
|
|
3269
|
-
* and shares nothing with any other take but the row's CAS. The reply is bound to the caller's own
|
|
3270
|
-
* request subtree, as `serveControl`'s `boundReply` binds it. */
|
|
3271
|
-
serveTakes(authority) {
|
|
3272
|
-
const sub = this.nc.subscribe(takeServeFilter(this.space), { queue: TAKE_ROUTE });
|
|
3273
|
-
this.subs.push(sub);
|
|
3274
|
-
void (async () => {
|
|
3275
|
-
for await (const m of sub) {
|
|
3276
|
-
if (!m.reply || !m.reply.startsWith(`${m.subject}.reply.`)) {
|
|
3277
|
-
this.emitRecoverable(new Error(`rejected take on ${m.subject}: reply target "${m.reply ?? "(none)"}" is not under the sender's own reply subtree`));
|
|
3278
|
-
continue;
|
|
3279
|
-
}
|
|
3280
|
-
void this.answerTake(authority, m).catch((e) => this.emitRecoverable(e));
|
|
3281
|
-
}
|
|
3282
|
-
})().catch((e) => this.emitRecoverable(e));
|
|
3283
|
-
return sub;
|
|
3284
|
-
}
|
|
3285
|
-
async answerTake(authority, m) {
|
|
3286
|
-
let reply;
|
|
3287
|
-
if (this.takesInFlight >= MAX_TAKES_IN_FLIGHT)
|
|
3288
|
-
reply = { ok: false, code: "busy", error: "the delivery daemon is answering too many takes; retry" };
|
|
3289
|
-
else {
|
|
3290
|
-
this.takesInFlight++;
|
|
3291
|
-
try {
|
|
3292
|
-
reply = await this.takeFor(authority, m);
|
|
3293
|
-
}
|
|
3294
|
-
catch (e) {
|
|
3295
|
-
reply = { ok: false, code: "busy", error: `the take could not be recorded: ${e.message}` };
|
|
3296
|
-
}
|
|
3297
|
-
finally {
|
|
3298
|
-
this.takesInFlight--;
|
|
3299
|
-
}
|
|
3300
|
-
}
|
|
3301
|
-
m.respond(JSON.stringify(reply));
|
|
3302
|
-
}
|
|
3303
|
-
/** Check one take's caller and arguments, then record it. The role and the caller come from the
|
|
3304
|
-
* broker-policed subject; the payload's `from.id` must match, as on every control rail. */
|
|
3305
|
-
async takeFor(authority, m) {
|
|
3306
|
-
if (!this.plane3MayAct())
|
|
3307
|
-
return { ok: false, code: "busy", error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
|
|
3308
|
-
const parsed = parseTakeSubject(m.subject);
|
|
3309
|
-
let req;
|
|
3310
|
-
try {
|
|
3311
|
-
req = m.json();
|
|
3312
|
-
}
|
|
3313
|
-
catch {
|
|
3314
|
-
req = undefined;
|
|
3315
|
-
}
|
|
3316
|
-
if (!parsed || !isPrincipalOwnerToken(parsed.owner) || req?.from?.id !== parsed.sender || typeof req.from.name !== "string")
|
|
3317
|
-
return { ok: false, code: "mismatch", error: "sender mismatch - take rejected" };
|
|
3318
|
-
if (!isAnycastSeq(req.seq) || !isAnycastSeq(req.count) || typeof req.id !== "string" || req.id.length > 256 ||
|
|
3319
|
-
typeof req.epoch !== "string" || req.epoch.length > 64)
|
|
3320
|
-
return { ok: false, code: "mismatch", error: "a take names a seq, the request id, a delivery count and the stream's creation time" };
|
|
3321
|
-
// The name reaches the sender's status text, so it must pass the rule a presence card name does.
|
|
3322
|
-
try {
|
|
3323
|
-
assertValidName(req.from.name);
|
|
3324
|
-
}
|
|
3325
|
-
catch (e) {
|
|
3326
|
-
return { ok: false, code: "mismatch", error: `a take names its holder by a valid display name: ${e.message}` };
|
|
3327
|
-
}
|
|
3328
|
-
return authority.take(parsed.role, { from: { id: parsed.sender, name: req.from.name }, seq: req.seq, id: req.id, count: req.count, epoch: req.epoch });
|
|
3329
|
-
}
|
|
3330
|
-
/** DM one notice to a request's sender from this daemon's own principal. The event key is the
|
|
3331
|
-
* message id, so a resend after a crash is dropped by the DM stream's duplicate window and by
|
|
3332
|
-
* the receiving connector. */
|
|
3333
|
-
async sendAnycastNotice(to, event) {
|
|
3334
|
-
if (!this.js)
|
|
3335
|
-
throw new Error(this.notLiveMsg());
|
|
3336
|
-
const recipient = parsePrincipalKey(to);
|
|
3337
|
-
if (!recipient)
|
|
3338
|
-
throw new Error(`anycast notice: "${to}" is not a principal`);
|
|
3339
|
-
const msg = {
|
|
3340
|
-
id: event.key,
|
|
3341
|
-
ts: Date.now(),
|
|
3342
|
-
space: this.space,
|
|
3343
|
-
from: this.ref(),
|
|
3344
|
-
to,
|
|
3345
|
-
parts: [{ kind: "text", text: event.text }, { kind: "data", data: { [ANYCAST_NOTICE_KEY]: event.notice } }],
|
|
3346
|
-
};
|
|
3347
|
-
await this.js.publish(unicastSubject(this.space, recipient.owner, recipient.actor, this.owner, this.actor), JSON.stringify(msg), { msgID: msg.id });
|
|
3348
|
-
}
|
|
3349
|
-
/** DM a post's sender that its mentions of `names` reached no one, because each of those agents
|
|
3350
|
-
* may not read `channel`. The id carries the post's stream seq, so a fan-out redelivered after a
|
|
3351
|
-
* crash sends the notice again under the same id and the DM stream drops the duplicate. */
|
|
3352
|
-
async sendMentionNotice(msg, channel, seq, names) {
|
|
3353
|
-
if (!this.js)
|
|
3354
|
-
throw new Error(this.notLiveMsg());
|
|
3355
|
-
const recipient = parsePrincipalKey(msg.from.id);
|
|
3356
|
-
if (!recipient)
|
|
3357
|
-
throw new Error(`mention notice: "${msg.from.id}" is not a principal`);
|
|
3358
|
-
const named = names.map((n) => `@${n}`).join(", ");
|
|
3359
|
-
const notice = {
|
|
3360
|
-
id: `mention.${seq}.${msg.id}`,
|
|
3361
|
-
ts: Date.now(),
|
|
3362
|
-
space: this.space,
|
|
3363
|
-
from: this.ref(),
|
|
3364
|
-
to: msg.from.id,
|
|
3365
|
-
parts: [
|
|
3366
|
-
{ kind: "text", text: `Your post on #${channel} mentioned ${named}, who may not read #${channel} and ${names.length === 1 ? "was" : "were"} not given it. Reach ${names.length === 1 ? "them" : "each"} another way, such as cotal_dm.` },
|
|
3367
|
-
{ kind: "data", data: { [MENTION_NOTICE_KEY]: { channel, message: msg.id, names: [...names] } } },
|
|
3368
|
-
],
|
|
3369
|
-
};
|
|
3370
|
-
await this.js.publish(unicastSubject(this.space, recipient.owner, recipient.actor, this.owner, this.actor), JSON.stringify(notice), { msgID: notice.id });
|
|
3371
|
-
}
|
|
3372
|
-
/** Answer a sender's status or cancel for one of its own requests. The caller is the
|
|
3373
|
-
* authenticated subject sender, checked against the request's stored subject. */
|
|
3374
|
-
async deliveryAnycast(op, caller, args) {
|
|
3375
|
-
const authority = this.anycastAuthority;
|
|
3376
|
-
if (!authority)
|
|
3377
|
-
return { ok: false, error: `delivery: ${this.anycastUnavailable ?? "the anycast request records are not open on this daemon; retry"}` };
|
|
3378
|
-
if (!isAnycastSeq(args.seq))
|
|
3379
|
-
return { ok: false, error: `${op}: seq must be a whole number of at least 1` };
|
|
3380
|
-
if (args.id !== undefined && (typeof args.id !== "string" || args.id.length > 256))
|
|
3381
|
-
return { ok: false, error: `${op}: id must be a request id` };
|
|
3382
|
-
const q = { seq: args.seq, ...(typeof args.id === "string" ? { id: args.id } : {}) };
|
|
3383
|
-
try {
|
|
3384
|
-
const status = op === ANYCAST_CANCEL_OP ? await authority.cancel(caller, q) : await authority.status(caller, q);
|
|
3385
|
-
if (!this.plane3MayAct())
|
|
3386
|
-
return { ok: false, error: "delivery: this daemon stopped serving this shard while the request was in flight; retry" };
|
|
3387
|
-
return { ok: true, data: status };
|
|
3388
|
-
}
|
|
3389
|
-
catch (e) {
|
|
3390
|
-
return { ok: false, error: e.message };
|
|
3391
|
-
}
|
|
3392
|
-
}
|
|
2625
|
+
/** The anycast request rail of this endpoint. Every dependency is read when it is used, so a field
|
|
2626
|
+
* assigned after construction is seen. */
|
|
2627
|
+
anycastRail = new EndpointAnycast({
|
|
2628
|
+
nc: () => this.nc,
|
|
2629
|
+
jsm: () => this.jsm,
|
|
2630
|
+
js: () => this.js,
|
|
2631
|
+
space: () => this.space,
|
|
2632
|
+
owner: () => this.owner,
|
|
2633
|
+
actor: () => this.actor,
|
|
2634
|
+
card: () => this.card,
|
|
2635
|
+
authed: () => this.authed,
|
|
2636
|
+
subs: () => this.subs,
|
|
2637
|
+
anycastUnavailable: () => this.anycastUnavailable,
|
|
2638
|
+
setAnycastUnavailable: (value) => { this.anycastUnavailable = value; },
|
|
2639
|
+
anycastAuthority: () => this.anycastAuthority,
|
|
2640
|
+
setAnycastAuthority: (value) => { this.anycastAuthority = value; },
|
|
2641
|
+
takeServeSub: () => this.takeServeSub,
|
|
2642
|
+
setTakeServeSub: (value) => { this.takeServeSub = value; },
|
|
2643
|
+
anycastSweep: () => this.anycastSweep,
|
|
2644
|
+
setAnycastSweep: (value) => { this.anycastSweep = value; },
|
|
2645
|
+
takesInFlight: () => this.takesInFlight,
|
|
2646
|
+
setTakesInFlight: (value) => { this.takesInFlight = value; },
|
|
2647
|
+
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
2648
|
+
plane3MayAct: () => this.deliveryPlane.plane3MayAct(),
|
|
2649
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
2650
|
+
ref: () => this.ref(),
|
|
2651
|
+
requestBoundReply: (nc, subject, body, reply, timeoutMs) => this.requestBoundReply(nc, subject, body, reply, timeoutMs),
|
|
2652
|
+
readTaskEpoch: () => this.consumers.readTaskEpoch(),
|
|
2653
|
+
requestDelivery: (op, args, timeoutMs) => this.requestDelivery(op, args, timeoutMs),
|
|
2654
|
+
});
|
|
3393
2655
|
// ---- the peer-readable liveness surface (#1577) --------------------------
|
|
3394
2656
|
/** The peer-readable liveness responders and the probe, of this endpoint. Every dependency is read
|
|
3395
2657
|
* when it is used, so a field assigned after construction is seen. */
|
|
@@ -3450,10 +2712,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3450
2712
|
serveLiveness(plane, readState) {
|
|
3451
2713
|
return this.liveness.serveLiveness(plane, readState);
|
|
3452
2714
|
}
|
|
3453
|
-
/** Forwards to the liveness component's {@link EndpointLiveness.bindLiveness}. */
|
|
3454
|
-
bindLiveness(plane) {
|
|
3455
|
-
return this.liveness.bindLiveness(plane);
|
|
3456
|
-
}
|
|
3457
2715
|
/** Bind the liveness responder for the DELIVERY plane, grading itself from its own shard-0 lease
|
|
3458
2716
|
* through the shared classifier. The daemon is the one process that can answer this honestly: it
|
|
3459
2717
|
* knows its own incarnation, so a predecessor's `ready:true` corpse in the bucket classifies
|
|
@@ -3542,134 +2800,42 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3542
2800
|
void this.armPrincipalServe(intent, nc, handle);
|
|
3543
2801
|
return { ready, failed, stop: () => this.stopPrincipalServe(intent) };
|
|
3544
2802
|
}
|
|
3545
|
-
/**
|
|
2803
|
+
/** The principal serve and prompts of this endpoint. Every dependency is read when it is used, and
|
|
2804
|
+
* every endpoint method is looked up at call time, so a member a caller replaced on the endpoint runs. */
|
|
2805
|
+
principalServe = new EndpointPrincipalServe({
|
|
2806
|
+
space: () => this.space,
|
|
2807
|
+
nc: () => this.nc,
|
|
2808
|
+
owner: () => this.owner,
|
|
2809
|
+
actor: () => this.actor,
|
|
2810
|
+
card: () => this.card,
|
|
2811
|
+
principalServes: () => this.principalServes,
|
|
2812
|
+
principalServers: () => principalServers,
|
|
2813
|
+
promptsPresence: () => this.promptsPresence,
|
|
2814
|
+
setPromptsPresence: (value) => { this.promptsPresence = value; },
|
|
2815
|
+
publishPresence: () => this.publishPresence(),
|
|
2816
|
+
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
2817
|
+
requireLifecycleUid: (what) => this.requireLifecycleUid(what),
|
|
2818
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
2819
|
+
});
|
|
2820
|
+
/** Forwards to the principal-serve component's {@link EndpointPrincipalServe.bindPrincipalHandle}. */
|
|
3546
2821
|
bindPrincipalHandle(intent, nc) {
|
|
3547
|
-
return
|
|
2822
|
+
return this.principalServe.bindPrincipalHandle(intent, nc);
|
|
3548
2823
|
}
|
|
3549
|
-
/**
|
|
3550
|
-
* subscription or a reply read. Only a reply publish is left, which no call or serve awaits. */
|
|
2824
|
+
/** Forwards to the principal-serve component's {@link EndpointPrincipalServe.ownedPrincipalRouteRefusal}. */
|
|
3551
2825
|
ownedPrincipalRouteRefusal(err) {
|
|
3552
|
-
|
|
3553
|
-
if (err.operation === "publish")
|
|
3554
|
-
return err.subject.startsWith(`${p}prin.`);
|
|
3555
|
-
return err.subject.startsWith(`${p}prin.`) || err.subject.startsWith(`${p}prin-reply.`);
|
|
2826
|
+
return this.principalServe.ownedPrincipalRouteRefusal(err);
|
|
3556
2827
|
}
|
|
3557
|
-
/**
|
|
3558
|
-
* connection shows the presence field. */
|
|
2828
|
+
/** Forwards to the principal-serve component's {@link EndpointPrincipalServe.armPrincipalServe}. */
|
|
3559
2829
|
async armPrincipalServe(intent, nc, handle) {
|
|
3560
|
-
intent
|
|
3561
|
-
// Every earlier binding is on a replaced connection: retire it once its handlers finish.
|
|
3562
|
-
for (const old of intent.handles)
|
|
3563
|
-
void old.stop().finally(() => {
|
|
3564
|
-
const i = intent.handles.indexOf(old);
|
|
3565
|
-
if (i >= 0)
|
|
3566
|
-
intent.handles.splice(i, 1);
|
|
3567
|
-
});
|
|
3568
|
-
intent.handles.push(handle);
|
|
3569
|
-
intent.readyOn = undefined;
|
|
3570
|
-
void handle.refused.then((err) => {
|
|
3571
|
-
if (!intent.ended && intent.binding?.handle === handle)
|
|
3572
|
-
this.endPrincipalServe(intent, err);
|
|
3573
|
-
});
|
|
3574
|
-
try {
|
|
3575
|
-
await handle.ready;
|
|
3576
|
-
}
|
|
3577
|
-
catch {
|
|
3578
|
-
// A refusal ends the serve through `refused`. A flush that failed because the connection
|
|
3579
|
-
// went away is answered by the binding the next connection makes.
|
|
3580
|
-
return;
|
|
3581
|
-
}
|
|
3582
|
-
this.principalServeConfirmed(intent, nc, handle);
|
|
2830
|
+
return this.principalServe.armPrincipalServe(intent, nc, handle);
|
|
3583
2831
|
}
|
|
3584
|
-
/**
|
|
3585
|
-
* ready there. */
|
|
3586
|
-
principalServeConfirmed(intent, nc, handle) {
|
|
3587
|
-
if (intent.ended || intent.binding?.handle !== handle || this.nc !== nc)
|
|
3588
|
-
return;
|
|
3589
|
-
intent.readyOn = nc;
|
|
3590
|
-
if (!intent.readySettled) {
|
|
3591
|
-
intent.readySettled = true;
|
|
3592
|
-
intent.resolveReady();
|
|
3593
|
-
}
|
|
3594
|
-
if (intent.serve.endpoint === AGENT_ENDPOINT)
|
|
3595
|
-
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
3596
|
-
}
|
|
3597
|
-
/** Re-bind every principal serve on a replaced connection (a rebuild), or re-run the check on the
|
|
3598
|
-
* connection the library reconnected and re-subscribed itself. */
|
|
2832
|
+
/** Forwards to the principal-serve component's {@link EndpointPrincipalServe.rebindPrincipalServes}. */
|
|
3599
2833
|
rebindPrincipalServes(reconnected) {
|
|
3600
|
-
|
|
3601
|
-
if (!nc)
|
|
3602
|
-
return;
|
|
3603
|
-
for (const intent of this.principalServes.values()) {
|
|
3604
|
-
if (intent.ended)
|
|
3605
|
-
continue;
|
|
3606
|
-
const bound = intent.binding;
|
|
3607
|
-
if (bound && bound.nc === nc) {
|
|
3608
|
-
if (!reconnected)
|
|
3609
|
-
continue;
|
|
3610
|
-
intent.readyOn = undefined;
|
|
3611
|
-
void bound.handle.check().then(() => this.principalServeConfirmed(intent, nc, bound.handle), () => { });
|
|
3612
|
-
continue;
|
|
3613
|
-
}
|
|
3614
|
-
let handle;
|
|
3615
|
-
try {
|
|
3616
|
-
handle = this.bindPrincipalHandle(intent, nc);
|
|
3617
|
-
}
|
|
3618
|
-
catch (e) {
|
|
3619
|
-
this.endPrincipalServe(intent, e);
|
|
3620
|
-
continue;
|
|
3621
|
-
}
|
|
3622
|
-
void this.armPrincipalServe(intent, nc, handle);
|
|
3623
|
-
}
|
|
2834
|
+
return this.principalServe.rebindPrincipalServes(reconnected);
|
|
3624
2835
|
}
|
|
3625
|
-
/**
|
|
3626
|
-
endPrincipalServe(intent, err) {
|
|
3627
|
-
if (intent.ended)
|
|
3628
|
-
return;
|
|
3629
|
-
this.forgetPrincipalServe(intent);
|
|
3630
|
-
const wasReady = intent.readySettled;
|
|
3631
|
-
if (!intent.readySettled) {
|
|
3632
|
-
intent.readySettled = true;
|
|
3633
|
-
intent.rejectReady(err);
|
|
3634
|
-
}
|
|
3635
|
-
intent.resolveFailed(err);
|
|
3636
|
-
for (const handle of intent.handles)
|
|
3637
|
-
void handle.stop();
|
|
3638
|
-
if (intent.serve.endpoint === AGENT_ENDPOINT) {
|
|
3639
|
-
this.promptsPresence = undefined;
|
|
3640
|
-
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
3641
|
-
}
|
|
3642
|
-
// `ready` already carried a refusal that came before it settled; a later one is reported here.
|
|
3643
|
-
if (wasReady)
|
|
3644
|
-
this.emitRecoverable(new Error(`stopped serving "${intent.serve.endpoint}" on the principal route: ${err.message}`, { cause: err }));
|
|
3645
|
-
}
|
|
3646
|
-
/** Mark `intent` ended and release what it holds in this endpoint and in this process. */
|
|
3647
|
-
forgetPrincipalServe(intent) {
|
|
3648
|
-
intent.ended = true;
|
|
3649
|
-
intent.readyOn = undefined;
|
|
3650
|
-
if (this.principalServes.get(intent.serve.endpoint) === intent)
|
|
3651
|
-
this.principalServes.delete(intent.serve.endpoint);
|
|
3652
|
-
if (principalServers.get(intent.serverKey) === intent)
|
|
3653
|
-
principalServers.delete(intent.serverKey);
|
|
3654
|
-
}
|
|
3655
|
-
/** Stop serving `intent` and wait for its handlers in flight, each until its request's deadline
|
|
3656
|
-
* at most. The ending is synchronous, so a
|
|
3657
|
-
* `ready` still waiting for the broker rejects before the caller's next step. `republish` writes
|
|
3658
|
-
* presence without the `prompts` field; the endpoint's own stop writes it offline instead. */
|
|
2836
|
+
/** Forwards to the principal-serve component's {@link EndpointPrincipalServe.stopPrincipalServe}. */
|
|
3659
2837
|
async stopPrincipalServe(intent, republish = true) {
|
|
3660
|
-
|
|
3661
|
-
this.forgetPrincipalServe(intent);
|
|
3662
|
-
if (!intent.readySettled) {
|
|
3663
|
-
intent.readySettled = true;
|
|
3664
|
-
intent.rejectReady(new Error(`stopped serving "${intent.serve.endpoint}" before the broker confirmed the subscription`));
|
|
3665
|
-
}
|
|
3666
|
-
if (intent.serve.endpoint === AGENT_ENDPOINT) {
|
|
3667
|
-
this.promptsPresence = undefined;
|
|
3668
|
-
if (republish)
|
|
3669
|
-
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
3670
|
-
}
|
|
3671
|
-
}
|
|
3672
|
-
await Promise.all(intent.handles.map((handle) => handle.stop()));
|
|
2838
|
+
return this.principalServe.stopPrincipalServe(intent, republish);
|
|
3673
2839
|
}
|
|
3674
2840
|
/**
|
|
3675
2841
|
* Set this process's `prompts` presence field and write presence at once (SPEC §6): how many
|
|
@@ -3680,18 +2846,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3680
2846
|
* endpoint is not served, and a RangeError for a `seq` below one this runtime already used.
|
|
3681
2847
|
*/
|
|
3682
2848
|
async setPromptsPresence(prompts) {
|
|
3683
|
-
|
|
3684
|
-
if (!Number.isSafeInteger(prompts?.[k]) || prompts[k] < 0)
|
|
3685
|
-
throw new TypeError(`prompts.${k} must be a non-negative safe integer`);
|
|
3686
|
-
const intent = this.principalServes.get(AGENT_ENDPOINT);
|
|
3687
|
-
if (!intent)
|
|
3688
|
-
throw new Error(`the prompts presence field belongs to a served "${AGENT_ENDPOINT}" endpoint: call servePrincipal first (SPEC §6)`);
|
|
3689
|
-
const used = promptsSeqs.get(intent.self.runtimeId) ?? 0;
|
|
3690
|
-
if (prompts.seq < used)
|
|
3691
|
-
throw new RangeError(`prompts.seq ${prompts.seq} is below ${used}, which this process already used under runtime ${intent.self.runtimeId}: seq never goes back under one runtime id (SPEC §6)`);
|
|
3692
|
-
promptsSeqs.set(intent.self.runtimeId, prompts.seq);
|
|
3693
|
-
this.promptsPresence = { open: prompts.open, seq: prompts.seq };
|
|
3694
|
-
await this.publishPresence();
|
|
2849
|
+
return this.principalServe.setPromptsPresence(prompts);
|
|
3695
2850
|
}
|
|
3696
2851
|
/**
|
|
3697
2852
|
* Take the next `prompts` `seq` for this process's runtime (SPEC §6): 0 the first time, then one
|
|
@@ -3699,18 +2854,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3699
2854
|
* prompt desk takes every `seq` it shows from here. Throws without a lifecycle UID.
|
|
3700
2855
|
*/
|
|
3701
2856
|
nextPromptsSeq() {
|
|
3702
|
-
|
|
3703
|
-
const used = promptsSeqs.get(runtimeId);
|
|
3704
|
-
const next = used === undefined ? 0 : used + 1;
|
|
3705
|
-
promptsSeqs.set(runtimeId, next);
|
|
3706
|
-
return next;
|
|
2857
|
+
return this.principalServe.nextPromptsSeq();
|
|
3707
2858
|
}
|
|
3708
|
-
/**
|
|
2859
|
+
/** Forwards to the principal-serve component's {@link EndpointPrincipalServe.servedPrompts}. */
|
|
3709
2860
|
servedPrompts() {
|
|
3710
|
-
|
|
3711
|
-
if (!intent || intent.ended || !this.nc || intent.readyOn !== this.nc)
|
|
3712
|
-
return undefined;
|
|
3713
|
-
return this.promptsPresence;
|
|
2861
|
+
return this.principalServe.servedPrompts();
|
|
3714
2862
|
}
|
|
3715
2863
|
/**
|
|
3716
2864
|
* Call one command of a principal's own endpoint on the principal route (SPEC 13.21), as this
|
|
@@ -3726,159 +2874,45 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3726
2874
|
* (`unknown`) when no reply came. See {@link epPrincipalCall}.
|
|
3727
2875
|
*/
|
|
3728
2876
|
async callPrincipal(call) {
|
|
3729
|
-
|
|
3730
|
-
if (!nc)
|
|
3731
|
-
throw new Error(this.notLiveMsg());
|
|
3732
|
-
const contract = call.contract
|
|
3733
|
-
?? (call.command === "describe" ? describeContracts() : call.endpoint === AGENT_ENDPOINT ? AGENT_CONTRACTS[call.command] : undefined);
|
|
3734
|
-
if (contract === undefined)
|
|
3735
|
-
throw new EpEnvelopeError("bad-request", `no contract is built in for ${call.endpoint}.${call.command}; pass one`, undefined, "not-executed");
|
|
3736
|
-
const caller = { owner: this.owner, actor: this.actor, uid: this.requireLifecycleUid("callPrincipal") };
|
|
3737
|
-
return epPrincipalCall(nc, this.space, { principal: call.principal, authority: call.authority }, {
|
|
3738
|
-
endpoint: call.endpoint, command: call.command, contract, caller, name: this.card.name,
|
|
3739
|
-
...(call.args !== undefined ? { args: call.args } : {}),
|
|
3740
|
-
}, { deadlineMs: call.deadlineMs, ...(call.signal ? { signal: call.signal } : {}) });
|
|
2877
|
+
return this.principalServe.callPrincipal(call);
|
|
3741
2878
|
}
|
|
3742
2879
|
/** Agent-side: request a Plane-3 durable backstop for a channel via the server-side delivery daemon (ctl.delivery). Throws
|
|
3743
2880
|
* when no privileged writer is present (open / no delivery daemon). 30s timeout — activation catch-up may
|
|
3744
2881
|
* run before the reply (the window is small, but a busy channel can take more than the 5s default). */
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
if (!reply.ok)
|
|
3748
|
-
throw new Error(reply.error ?? "durable join rejected");
|
|
3749
|
-
return reply.data ?? { durable: false };
|
|
2882
|
+
durableJoinChannel(channel) {
|
|
2883
|
+
return this.durable.durableJoinChannel(channel);
|
|
3750
2884
|
}
|
|
3751
2885
|
/** Agent-side: release a Plane-3 durable backstop (tombstone membership at the leave cursor). Passes
|
|
3752
2886
|
* the join generation so a stale leave can't tombstone a newer rejoin (the delivery daemon validates
|
|
3753
2887
|
* it) AND this incarnation's lifecycleUid — membership rows are lifecycle-keyed (SPEC 13.1), and a
|
|
3754
2888
|
* leave must resolve the selected principal-owned row even after the ACL row was narrowed or purged. */
|
|
3755
|
-
|
|
3756
|
-
|
|
3757
|
-
if (!reply.ok)
|
|
3758
|
-
throw new Error(reply.error ?? "durable leave rejected");
|
|
3759
|
-
}
|
|
3760
|
-
/** Fail-closed async cleanup for a channel forced out by a LATE sub.allow refusal (the broker revoked
|
|
3761
|
-
* the live read). The sync sub callback can't await, so this RETRIES the Plane-3 tombstone with capped
|
|
3762
|
-
* backoff UNTIL IT SUCCEEDS (or the endpoint stops) — the §7 boundary always closes once the manager
|
|
3763
|
-
* is reachable, never a silent give-up. While pending, the channel is tracked in
|
|
3764
|
-
* {@link pendingDurableLeave} and surfaced via {@link pendingDurableLeaves} (the connector shows it in
|
|
3765
|
-
* `cotal_channels` as `durable-unclosed`, never ordinary absence). The generation is kept the whole
|
|
3766
|
-
* time. Authoritative closure of a revoked membership is also handled by revocation (rotate creds + tear down). */
|
|
3767
|
-
async closeRefusedMembership(channel, generation) {
|
|
3768
|
-
this.pendingDurableLeave.set(channel, generation);
|
|
3769
|
-
for (let attempt = 0;; attempt++) {
|
|
3770
|
-
if (this.stopped)
|
|
3771
|
-
return;
|
|
3772
|
-
try {
|
|
3773
|
-
await this.durableLeaveChannel(channel, generation);
|
|
3774
|
-
this.plane3Channels.delete(channel);
|
|
3775
|
-
this.pendingDurableLeave.delete(channel);
|
|
3776
|
-
return;
|
|
3777
|
-
}
|
|
3778
|
-
catch (e) {
|
|
3779
|
-
if (attempt === 0)
|
|
3780
|
-
this.emitRecoverable(new Error(`channel "${channel}": Plane-3 durable membership (generation ${generation}) not yet tombstoned after a refused live sub - retrying; §7 boundary may be open until it succeeds (${e.message})`));
|
|
3781
|
-
await this.membershipPause(attempt);
|
|
3782
|
-
}
|
|
3783
|
-
}
|
|
2889
|
+
durableLeaveChannel(channel, generation, lifecycleUid) {
|
|
2890
|
+
return this.durable.durableLeaveChannel(channel, generation, lifecycleUid);
|
|
3784
2891
|
}
|
|
3785
2892
|
/** Channels with a Plane-3 durable membership whose §7 tombstone is still pending after a refused live
|
|
3786
|
-
* sub (see {@link closeRefusedMembership}) — surfaced by the connector as a `durable-unclosed` state so
|
|
2893
|
+
* sub (see {@link EndpointDurable.closeRefusedMembership}) — surfaced by the connector as a `durable-unclosed` state so
|
|
3787
2894
|
* it is never presented as ordinary "not subscribed". */
|
|
3788
2895
|
pendingDurableLeaves() {
|
|
3789
|
-
return
|
|
2896
|
+
return this.durable.pendingDurableLeaves();
|
|
3790
2897
|
}
|
|
3791
|
-
/**
|
|
3792
|
-
* distinct from a responder that errored. nats.js surfaces it as NoRespondersError, or a RequestError
|
|
3793
|
-
* whose `isNoResponders()` is true. */
|
|
2898
|
+
/** Forwards to the anycast component's {@link EndpointAnycast.isNoResponders}. */
|
|
3794
2899
|
isNoResponders(e) {
|
|
3795
|
-
return
|
|
2900
|
+
return this.anycastRail.isNoResponders(e);
|
|
3796
2901
|
}
|
|
3797
|
-
/**
|
|
3798
|
-
|
|
3799
|
-
|
|
3800
|
-
async takeAnycast(role, seq, id, count, epoch, nak) {
|
|
3801
|
-
const nc = this.nc;
|
|
3802
|
-
if (!nc)
|
|
3803
|
-
return { taken: false, drop: false, reason: "daemon-down", detail: this.notLiveMsg() };
|
|
3804
|
-
const held = (reason, detail) => ({ taken: false, drop: false, reason, detail });
|
|
3805
|
-
const subject = takeSubject(this.space, role, this.owner, this.actor);
|
|
3806
|
-
const body = { from: { id: this.card.id, name: this.card.name }, seq, id, count, epoch };
|
|
3807
|
-
let reply;
|
|
3808
|
-
try {
|
|
3809
|
-
const m = await this.requestBoundReply(nc, subject, JSON.stringify(body), `${subject}.reply.${randomUUID()}`, ANYCAST_TAKE_TIMEOUT_MS);
|
|
3810
|
-
reply = m.json();
|
|
3811
|
-
}
|
|
3812
|
-
catch (e) {
|
|
3813
|
-
if (isPermissionDenied(e))
|
|
3814
|
-
return held("credential-outdated", "this seat's credential was minted before anycast takes existed, so it cannot take role requests; respawn the seat to get a current credential");
|
|
3815
|
-
if (this.isNoResponders(e))
|
|
3816
|
-
return this.takeResponderMissing();
|
|
3817
|
-
return held("daemon-down", `the delivery daemon did not answer the take: ${e.message}`);
|
|
3818
|
-
}
|
|
3819
|
-
if (reply.ok)
|
|
3820
|
-
return { taken: true };
|
|
3821
|
-
// A mismatch can mean the stream was recreated under this consumer, which resumes on the new
|
|
3822
|
-
// stream: read its creation time again, so the copies delivered from here on carry it. When the
|
|
3823
|
-
// time moved, this copy is nakked, so the broker delivers it again at once with the new time. A
|
|
3824
|
-
// redelivery that still mismatches carries the time read here, so it is forgotten, not nakked.
|
|
3825
|
-
if (reply.code === "mismatch") {
|
|
3826
|
-
const current = await this.readTaskEpoch().catch(() => undefined);
|
|
3827
|
-
if (current !== undefined && current !== epoch)
|
|
3828
|
-
nak();
|
|
3829
|
-
}
|
|
3830
|
-
if (reply.code === "cancelled" || reply.code === "taken" || reply.code === "gone" || reply.code === "mismatch")
|
|
3831
|
-
return { taken: false, drop: true, reason: reply.code, detail: reply.error };
|
|
3832
|
-
return held("daemon-busy", reply.error);
|
|
3833
|
-
}
|
|
3834
|
-
/** No responder on the take rail: an older daemon answers the probe op as unsupported, and a
|
|
3835
|
-
* stopped one does not answer at all. */
|
|
3836
|
-
async takeResponderMissing() {
|
|
3837
|
-
const held = (reason, detail) => ({ taken: false, drop: false, reason, detail });
|
|
3838
|
-
let reply;
|
|
3839
|
-
try {
|
|
3840
|
-
reply = await this.requestDelivery(ANYCAST_TAKES_OP, {}, ANYCAST_TAKE_TIMEOUT_MS);
|
|
3841
|
-
}
|
|
3842
|
-
catch {
|
|
3843
|
-
return held("daemon-down", "no delivery daemon is answering on this mesh, so role requests wait on the queue until one does");
|
|
3844
|
-
}
|
|
3845
|
-
if (!reply.ok && /not supported on the delivery control service/.test(reply.error ?? ""))
|
|
3846
|
-
return held("daemon-outdated", "the delivery daemon predates anycast takes, so this seat holds role requests until the daemon is restarted at this version (`cotal up` runs it)");
|
|
3847
|
-
if (!reply.ok && reply.error?.includes(ANYCAST_RECORDS_CREDENTIAL_OUTDATED))
|
|
3848
|
-
return held("daemon-outdated", `this seat holds role requests until the delivery daemon can record takes: ${ANYCAST_RECORDS_CREDENTIAL_OUTDATED}`);
|
|
3849
|
-
return held("daemon-down", "the delivery daemon is not answering takes, so role requests wait on the queue until it does");
|
|
2902
|
+
/** Forwards to the anycast component's {@link EndpointAnycast.takeAnycast}. */
|
|
2903
|
+
takeAnycast(role, seq, id, count, epoch, nak) {
|
|
2904
|
+
return this.anycastRail.takeAnycast(role, seq, id, count, epoch, nak);
|
|
3850
2905
|
}
|
|
3851
2906
|
/** The status of one of this endpoint's own anycast requests, from the delivery daemon's record
|
|
3852
2907
|
* or, with no record, from the queue (SPEC §8.1). `id` narrows the answer to that request. Throws
|
|
3853
2908
|
* on an open mesh, which runs no delivery daemon. */
|
|
3854
|
-
|
|
3855
|
-
return this.
|
|
2909
|
+
anycastStatus(seq, id) {
|
|
2910
|
+
return this.anycastRail.anycastStatus(seq, id);
|
|
3856
2911
|
}
|
|
3857
2912
|
/** Cancel one of this endpoint's own anycast requests while it waits. A request a holder already
|
|
3858
2913
|
* took is refused, and the returned status names the taker. */
|
|
3859
|
-
|
|
3860
|
-
return this.
|
|
3861
|
-
}
|
|
3862
|
-
async anycastRequest(op, seq, id) {
|
|
3863
|
-
if (!this.authed)
|
|
3864
|
-
throw new Error("an open mesh runs no delivery daemon, so it keeps no record of anycast requests");
|
|
3865
|
-
if (!isAnycastSeq(seq))
|
|
3866
|
-
throw new Error(`seq must be a whole number of at least 1, got ${seq}`);
|
|
3867
|
-
let reply;
|
|
3868
|
-
try {
|
|
3869
|
-
reply = await this.requestDelivery(op, { seq, ...(id !== undefined ? { id } : {}) }, 10_000);
|
|
3870
|
-
}
|
|
3871
|
-
catch (e) {
|
|
3872
|
-
if (this.isNoResponders(e))
|
|
3873
|
-
throw new Error("no delivery daemon is answering on this mesh, so the request's record cannot be read");
|
|
3874
|
-
throw e;
|
|
3875
|
-
}
|
|
3876
|
-
if (!reply.ok) {
|
|
3877
|
-
if (/not supported on the delivery control service/.test(reply.error ?? ""))
|
|
3878
|
-
throw new Error("the delivery daemon predates anycast request status; restart it at this version (`cotal up` runs it)");
|
|
3879
|
-
throw new Error(reply.error ?? `${op} failed`);
|
|
3880
|
-
}
|
|
3881
|
-
return reply.data;
|
|
2914
|
+
cancelAnycast(seq, id) {
|
|
2915
|
+
return this.anycastRail.cancelAnycast(seq, id);
|
|
3882
2916
|
}
|
|
3883
2917
|
/** The boot durable self-join and its reconcile, and the membership read the leave path uses, of
|
|
3884
2918
|
* this endpoint. Every dependency is read when it is used, and every endpoint method is looked up
|
|
@@ -3892,7 +2926,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3892
2926
|
requireLifecycleUid: (what) => this.requireLifecycleUid(what),
|
|
3893
2927
|
deliveryClassFresh: (channel) => this.deliveryClassFresh(channel),
|
|
3894
2928
|
durableJoinChannel: (channel) => this.durableJoinChannel(channel),
|
|
3895
|
-
membershipPause: (attempt) => this.membershipPause(attempt),
|
|
2929
|
+
membershipPause: (attempt) => this.reconnection.membershipPause(attempt),
|
|
3896
2930
|
isNoResponders: (e) => this.isNoResponders(e),
|
|
3897
2931
|
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
3898
2932
|
});
|
|
@@ -3903,398 +2937,55 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3903
2937
|
async fetchMemberships(channel) {
|
|
3904
2938
|
return this.bootMemberships.fetchMemberships(channel);
|
|
3905
2939
|
}
|
|
3906
|
-
/** Forwards to the boot-membership component's {@link EndpointBootMemberships.armBootDurableMemberships}. */
|
|
3907
|
-
async armBootDurableMemberships() {
|
|
3908
|
-
return this.bootMemberships.armBootDurableMemberships();
|
|
3909
|
-
}
|
|
3910
2940
|
/** True if this session holds an established Plane-3 durable membership for `channel` (in `plane3Channels`).
|
|
3911
2941
|
* Drives the membership-aware delivery-health surface: a joined durable channel that is NOT yet a member
|
|
3912
2942
|
* (boot self-join pending / daemon down) must render degraded, never "active" off a live lease alone. */
|
|
3913
2943
|
hasDurableMembership(channel) {
|
|
3914
2944
|
return this.bootMemberships.hasDurableMembership(channel);
|
|
3915
2945
|
}
|
|
3916
|
-
/**
|
|
3917
|
-
*
|
|
3918
|
-
|
|
3919
|
-
|
|
3920
|
-
|
|
3921
|
-
this.jsm
|
|
3922
|
-
|
|
3923
|
-
|
|
3924
|
-
|
|
3925
|
-
|
|
3926
|
-
|
|
3927
|
-
|
|
3928
|
-
|
|
3929
|
-
|
|
3930
|
-
|
|
3931
|
-
|
|
3932
|
-
|
|
3933
|
-
|
|
3934
|
-
|
|
3935
|
-
|
|
3936
|
-
|
|
3937
|
-
|
|
3938
|
-
|
|
3939
|
-
|
|
3940
|
-
|
|
3941
|
-
|
|
3942
|
-
|
|
3943
|
-
|
|
3944
|
-
|
|
3945
|
-
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
// self-join, the delivery daemon's fan-out writer + trusted reader deliver via the `dlv_<id>` pump
|
|
3960
|
-
// above; open dev mode is live-only — the durable plane needs the daemon's trusted reader, the
|
|
3961
|
-
// security boundary). Per-
|
|
3962
|
-
// channel history is the explicit replay-gated backfill, on FIRST connect only; a reconnect reopens
|
|
3963
|
-
// the subs without re-backfilling (the durable backstop redelivers any missed window via dlv).
|
|
3964
|
-
if (this.channels.length) {
|
|
3965
|
-
// Arm the per-channel join watermarks BEFORE opening the subs: the backfill reads <= frontier and
|
|
3966
|
-
// the core-sub delivers > frontier, so they never overlap (first connect). On reconnect we reopen
|
|
3967
|
-
// without arming/backfilling.
|
|
3968
|
-
const armed = this.firstConnect ? await this.armJoin(this.channels) : undefined;
|
|
3969
|
-
for (const ch of this.channels)
|
|
3970
|
-
this.subscribeChat(ch);
|
|
3971
|
-
await this.confirmChatSub();
|
|
3972
|
-
if (armed)
|
|
3973
|
-
await this.backfillArmed(armed, this.backfillFloor);
|
|
3974
|
-
}
|
|
3975
|
-
// First connect, auth mode: self-join BOOT durable channels via the server-side delivery daemon
|
|
3976
|
-
// (it owns membership now — there is no manager-written boot membership). Seeds plane3Channels so a
|
|
3977
|
-
// later leave can tombstone the §7 boundary; idempotent on relaunch. Open mode has no Plane-3.
|
|
3978
|
-
if (this.firstConnect && this.authed && this.channels.length)
|
|
3979
|
-
await this.armBootDurableMemberships();
|
|
3980
|
-
this.firstConnect = false;
|
|
3981
|
-
// Anycast: a shared work-queue consumer for our role — one instance grabs each task.
|
|
3982
|
-
// Open mode self-creates; auth mode BINDS the provisioner-pre-created svc_<role>
|
|
3983
|
-
// durable (agents are denied CONSUMER.CREATE on TASK_<space>, since the create-time
|
|
3984
|
-
// filter is the cross-role-drain attack surface — see provisionTaskQueue).
|
|
3985
|
-
if (this.card.role) {
|
|
3986
|
-
if (!this.authed) {
|
|
3987
|
-
await this.jsm.consumers.add(taskStream(this.space), taskDurableConfig(this.space, this.card.role, { ackWaitMs: this.ackWaitMs }));
|
|
3988
|
-
}
|
|
3989
|
-
await this.pump(taskStream(this.space), taskDurable(this.card.role), this.authed ? { takeRole: this.card.role } : {});
|
|
3990
|
-
}
|
|
3991
|
-
}
|
|
3992
|
-
/** Drive one consumer: decode and hand each message to listeners with ack control. Our own sends are
|
|
3993
|
-
* delivered too: on the DM inbox and the role queue they were addressed to us, so none is an echo,
|
|
3994
|
-
* and acking one unseen on the work queue would delete the only copy of the request. */
|
|
3995
|
-
async pump(stream, durable, opts = {}) {
|
|
3996
|
-
if (!this.js)
|
|
3997
|
-
throw new Error(this.notLiveMsg());
|
|
3998
|
-
const consumer = opts.takeRole === undefined
|
|
3999
|
-
? await this.js.consumers.get(stream, durable)
|
|
4000
|
-
: await this.bindTaskConsumer(stream, durable);
|
|
4001
|
-
const ackWaitMs = await ackWaitOf(consumer);
|
|
4002
|
-
const msgs = await consumer.consume();
|
|
4003
|
-
this.streamMsgs.push(msgs);
|
|
4004
|
-
if (opts.inbox)
|
|
4005
|
-
this.watchInbox(msgs, stream, durable);
|
|
4006
|
-
void (async () => {
|
|
4007
|
-
for await (const m of msgs) {
|
|
4008
|
-
const authed = this.authenticateStreamMsg(m);
|
|
4009
|
-
if (!authed)
|
|
4010
|
-
continue;
|
|
4011
|
-
const { msg, parsed } = authed;
|
|
4012
|
-
const delivery = durableDelivery(m, ackWaitMs);
|
|
4013
|
-
// On an auth mesh a role request carries its take: the holder records it with the delivery
|
|
4014
|
-
// daemon before showing the request, so its sender can cancel it until then (SPEC §8.1).
|
|
4015
|
-
// The copy carries the creation time of the stream it came from, read at bind.
|
|
4016
|
-
const role = opts.takeRole;
|
|
4017
|
-
if (role !== undefined) {
|
|
4018
|
-
const epoch = this.taskEpoch;
|
|
4019
|
-
delivery.take = () => this.takeAnycast(role, m.seq, msg.id, m.info.deliveryCount, epoch, () => m.nak());
|
|
4020
|
-
}
|
|
4021
|
-
this.emit("message", authenticatedMessage(msg, parsed), delivery, {
|
|
4022
|
-
historical: false,
|
|
4023
|
-
kind: kindFromParsed(parsed.kind),
|
|
4024
|
-
});
|
|
4025
|
-
}
|
|
4026
|
-
})().catch((e) => {
|
|
4027
|
-
if (!this.stopped)
|
|
4028
|
-
this.emit("error", e);
|
|
4029
|
-
});
|
|
4030
|
-
}
|
|
4031
|
-
/** Bind the role consumer and read the TASK stream's creation time on both sides of the bind, so
|
|
4032
|
-
* the time belongs to the stream the consumer was bound on (SPEC §8.1). */
|
|
4033
|
-
async bindTaskConsumer(stream, durable) {
|
|
4034
|
-
for (let attempt = 0; attempt < 3; attempt++) {
|
|
4035
|
-
const before = await this.readTaskEpoch();
|
|
4036
|
-
const consumer = await this.js.consumers.get(stream, durable);
|
|
4037
|
-
if (before === (await this.readTaskEpoch()))
|
|
4038
|
-
return consumer;
|
|
4039
|
-
}
|
|
4040
|
-
throw new Error(`the ${stream} stream was recreated while its role consumer was binding; reconnect to bind it again`);
|
|
4041
|
-
}
|
|
4042
|
-
/** Read the TASK stream's creation time into {@link taskEpoch}. */
|
|
4043
|
-
async readTaskEpoch() {
|
|
4044
|
-
const epoch = (await (await this.manager()).streams.info(taskStream(this.space))).created;
|
|
4045
|
-
this.taskEpoch = epoch;
|
|
4046
|
-
return epoch;
|
|
4047
|
-
}
|
|
4048
|
-
/** Warn once when the broker reports this endpoint's own DM inbox missing while the connection is
|
|
4049
|
-
* up, as after a store reset that the client's reconnect rode through, or after a manager restart
|
|
4050
|
-
* deletes the inbox. nats.js keeps checking and resumes pulling if the inbox comes back, so this
|
|
4051
|
-
* only reports. A `consumer_deleted` status is the consumer going away, so it warns at once. A
|
|
4052
|
-
* not-found can be a passing state, so that warning waits for a second report; the several 409s a
|
|
4053
|
-
* delete answers pending pulls with count once. A pull means the consumer was found again, which
|
|
4054
|
-
* re-arms the warning. The loop ends when the consumer is stopped, by `stop()` or by a rebuild
|
|
4055
|
-
* replacing this connection. */
|
|
4056
|
-
watchInbox(msgs, stream, durable) {
|
|
4057
|
-
const boundNc = this.nc;
|
|
4058
|
-
void (async () => {
|
|
4059
|
-
let absences = 0;
|
|
4060
|
-
let deleted = false;
|
|
4061
|
-
for await (const s of msgs.status()) {
|
|
4062
|
-
if (this.stopped || this.nc !== boundNc)
|
|
4063
|
-
break;
|
|
4064
|
-
if (s.type === "next") {
|
|
4065
|
-
absences = 0;
|
|
4066
|
-
deleted = false;
|
|
4067
|
-
continue;
|
|
4068
|
-
}
|
|
4069
|
-
if (s.type === "consumer_deleted") {
|
|
4070
|
-
if (deleted)
|
|
4071
|
-
continue;
|
|
4072
|
-
deleted = true;
|
|
4073
|
-
absences = 1; // the deletion itself is the second report: warn on this one
|
|
4074
|
-
}
|
|
4075
|
-
else if (s.type !== "consumer_not_found" && s.type !== "stream_not_found")
|
|
4076
|
-
continue;
|
|
4077
|
-
if (++absences === 2)
|
|
4078
|
-
this.emitRecoverable(new Error(`DM inbox missing: the broker reports consumer "${durable}" on stream "${stream}" missing, so direct ` +
|
|
4079
|
-
"messages to this endpoint are not delivered. The endpoint keeps checking; if the inbox does not " +
|
|
4080
|
-
"come back, end this session and start a new one"));
|
|
4081
|
-
}
|
|
4082
|
-
})().catch((e) => {
|
|
4083
|
-
if (!this.stopped)
|
|
4084
|
-
this.emit("error", e);
|
|
4085
|
-
});
|
|
4086
|
-
}
|
|
4087
|
-
/** Decode one stream copy and apply the authenticity guard (fail closed): the sender is encoded in
|
|
4088
|
-
* the subject, which the server policed who could publish. The payload `from` is advisory — it
|
|
4089
|
-
* must match, and a missing `from` or an unparseable subject on a delivery is itself an anomaly.
|
|
4090
|
-
* A rejected copy is `term()`ed (permanently invalid, never redelivered) and reported, and
|
|
4091
|
-
* `undefined` returns, so no handler ever sees it. */
|
|
4092
|
-
authenticateStreamMsg(m) {
|
|
4093
|
-
let msg;
|
|
4094
|
-
try {
|
|
4095
|
-
msg = m.json();
|
|
4096
|
-
}
|
|
4097
|
-
catch (e) {
|
|
4098
|
-
m.term(); // undecodable — never redeliver
|
|
4099
|
-
this.emit("error", e);
|
|
4100
|
-
return undefined;
|
|
4101
|
-
}
|
|
4102
|
-
if (!isRecord(msg) || !isUsableMessageId(msg.id)) {
|
|
4103
|
-
m.term(); // malformed envelope (SPEC sec 5): absent/non-string id — permanently invalid
|
|
4104
|
-
this.emit("error", new Error(`dropped message on ${m.subject}: absent or non-string id`));
|
|
4105
|
-
return undefined;
|
|
4106
|
-
}
|
|
4107
|
-
const parsed = parseSubject(m.subject);
|
|
4108
|
-
if (!parsed || !isFromSubjectSender(msg, parsed)) {
|
|
4109
|
-
m.term();
|
|
4110
|
-
this.emit("error", new Error(`dropped message on ${m.subject}: payload from ${msg.from?.id ?? "(none)"} ` +
|
|
4111
|
-
`does not match subject sender ${parsed?.sender ?? "(unparseable)"}`));
|
|
4112
|
-
return undefined;
|
|
4113
|
-
}
|
|
4114
|
-
return { msg, parsed };
|
|
4115
|
-
}
|
|
4116
|
-
/** Bind this endpoint's own existing DM durable and drive it serially (EndpointOptions.dmOnly).
|
|
4117
|
-
* The bind is one CONSUMER.INFO on `dm_<owner>-<actor>-<uid>`; reading is MSG.NEXT with a batch
|
|
4118
|
-
* of one, and the next copy reaches the handler only after the previous handler settles. */
|
|
4119
|
-
async bindDmOnly(mode) {
|
|
4120
|
-
if (!this.js)
|
|
4121
|
-
throw new Error(this.notLiveMsg());
|
|
4122
|
-
const stream = dmStream(this.space);
|
|
4123
|
-
const durable = dmDurable(this.owner, this.actor, this.requireLifecycleUid("consuming the DM inbox"));
|
|
4124
|
-
let consumer;
|
|
4125
|
-
try {
|
|
4126
|
-
consumer = await this.js.consumers.get(stream, durable);
|
|
4127
|
-
}
|
|
4128
|
-
catch (e) {
|
|
4129
|
-
if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
|
|
4130
|
-
throw new DmDurableMissingError(stream, durable);
|
|
4131
|
-
throw new DmBindRefusedError(stream, durable, e);
|
|
4132
|
-
}
|
|
4133
|
-
const ackWaitMs = await ackWaitOf(consumer); // cached by the bind's own CONSUMER.INFO, no request
|
|
4134
|
-
// A rebind after reconnect must not overlap the previous epoch's running handler. A handler that
|
|
4135
|
-
// does not settle within the stop bound fails this bind; the retry loop tries again.
|
|
4136
|
-
if (this.dmOnlyInFlight) {
|
|
4137
|
-
const settleMs = mode.settleTimeoutMs ?? 30_000;
|
|
4138
|
-
await withDeadline(this.dmOnlyInFlight, settleMs, () => new Error(`DM-only rebind waited ${settleMs}ms for the previous handler to settle; not reading the next DM yet`));
|
|
4139
|
-
}
|
|
4140
|
-
const msgs = await consumer.consume({ max_messages: 1 });
|
|
4141
|
-
this.streamMsgs.push(msgs);
|
|
4142
|
-
// The epoch this loop belongs to. nats.js can yield copies it already received after stop() began
|
|
4143
|
-
// or after a rebuild replaced the connection (a stacked pull after a missed-heartbeat reset hands
|
|
4144
|
-
// over several at once). Such a copy never reaches the handler and is neither acked nor nak'd:
|
|
4145
|
-
// the connection is closing or closed, so any reply could be lost, and the broker redelivers an
|
|
4146
|
-
// unacked copy after the durable's ack_wait to whichever epoch binds next.
|
|
4147
|
-
const boundNc = this.nc;
|
|
4148
|
-
const retired = () => this.stopped || this.nc !== boundNc || !boundNc || boundNc.isClosed();
|
|
4149
|
-
void (async () => {
|
|
4150
|
-
for await (const m of msgs) {
|
|
4151
|
-
if (retired())
|
|
4152
|
-
break;
|
|
4153
|
-
const authed = this.authenticateStreamMsg(m);
|
|
4154
|
-
if (!authed)
|
|
4155
|
-
continue;
|
|
4156
|
-
if (authed.parsed.kind !== "inst") {
|
|
4157
|
-
m.term(); // the DM stream filter admits only inst.* subjects; anything else is not a DM
|
|
4158
|
-
continue;
|
|
4159
|
-
}
|
|
4160
|
-
const run = Promise.resolve().then(() => mode.handler(authenticatedMessage(authed.msg, authed.parsed), dmOnlyDelivery(m, ackWaitMs, boundNc), {
|
|
4161
|
-
historical: false,
|
|
4162
|
-
kind: "dm",
|
|
4163
|
-
}));
|
|
4164
|
-
const flight = run.then(() => { }, () => { });
|
|
4165
|
-
this.dmOnlyInFlight = flight;
|
|
4166
|
-
try {
|
|
4167
|
-
await run;
|
|
4168
|
-
}
|
|
4169
|
-
catch (e) {
|
|
4170
|
-
this.emit("error", e instanceof Error ? e : new Error(String(e)));
|
|
4171
|
-
}
|
|
4172
|
-
finally {
|
|
4173
|
-
if (this.dmOnlyInFlight === flight)
|
|
4174
|
-
this.dmOnlyInFlight = undefined;
|
|
4175
|
-
}
|
|
4176
|
-
}
|
|
4177
|
-
})().catch((e) => {
|
|
4178
|
-
if (!this.stopped)
|
|
4179
|
-
this.emit("error", e);
|
|
4180
|
-
});
|
|
4181
|
-
}
|
|
4182
|
-
/** Open a native core subscription to a channel's live feed (the manager-free live read path,
|
|
4183
|
-
* broker-enforced by `sub.allow`). At-most-once — no replay, no ack; it is the live delivery for
|
|
4184
|
-
* every channel (boot + runtime). For a `durable` channel it is also the low-latency wake-hint
|
|
4185
|
-
* alongside the Plane-3 durable copy, coalesced by the receiver's id-dedup. Drops our own echo +
|
|
4186
|
-
* spoofed senders. */
|
|
4187
|
-
subscribeChat(channel) {
|
|
4188
|
-
if (!this.nc || this.chatSubs.has(channel))
|
|
4189
|
-
return;
|
|
4190
|
-
this.chatSubDenied.delete(channel);
|
|
4191
|
-
const subject = chatSubject(this.space, "*", "*", channel);
|
|
4192
|
-
const sub = this.nc.subscribe(subject, {
|
|
4193
|
-
callback: (err, m) => {
|
|
4194
|
-
if (err) {
|
|
4195
|
-
// async sub.allow refusal (or sub error): the live feed for this channel is dead — never a
|
|
4196
|
-
// leak (the broker refused it). Drop the channel from local joined state even if it was
|
|
4197
|
-
// already treated as joined — a LATE refusal beyond the confirm window: conformance #13
|
|
4198
|
-
// "drop on late refusal". (During the join's own confirm the channel isn't pushed yet, so
|
|
4199
|
-
// this fires nothing then; joinChannel reads `chatSubDenied` and throws cleanly.) Either way
|
|
4200
|
-
// the refusal is reported from here, by that throw or the leave below, so watchStatus skips it.
|
|
4201
|
-
if (err instanceof PermissionViolationError)
|
|
4202
|
-
this.chatSubRefusals.add(err);
|
|
4203
|
-
this.chatSubDenied.add(channel);
|
|
4204
|
-
this.chatSubs.delete(channel);
|
|
4205
|
-
const i = this.channels.indexOf(channel);
|
|
4206
|
-
if (i >= 0) {
|
|
4207
|
-
this.channels.splice(i, 1);
|
|
4208
|
-
// A late sub.allow refusal forces this agent out of the channel (the broker revoked its live
|
|
4209
|
-
// read). If it held a Plane-3 durable membership, the §7 boundary must close too. This sub
|
|
4210
|
-
// callback can't await, so a fail-closed async helper RETRIES the tombstone (backoff) UNTIL it
|
|
4211
|
-
// succeeds, clearing the mirror only then; while pending it is surfaced via cotal_channels —
|
|
4212
|
-
// never a silent drop, never lost retry state.
|
|
4213
|
-
const gen = this.plane3Channels.get(channel);
|
|
4214
|
-
if (gen !== undefined)
|
|
4215
|
-
void this.closeRefusedMembership(channel, gen);
|
|
4216
|
-
this.emit("error", new Error(`left channel "${channel}": its live subscription was refused by the broker`, { cause: err }));
|
|
4217
|
-
}
|
|
4218
|
-
return;
|
|
4219
|
-
}
|
|
4220
|
-
const parsed = parseSubject(m.subject);
|
|
4221
|
-
if (!parsed || parsed.kind !== "chat")
|
|
4222
|
-
return;
|
|
4223
|
-
let msg;
|
|
4224
|
-
try {
|
|
4225
|
-
msg = m.json();
|
|
4226
|
-
}
|
|
4227
|
-
catch (e) {
|
|
4228
|
-
this.emit("error", e);
|
|
4229
|
-
return;
|
|
4230
|
-
}
|
|
4231
|
-
if (!isRecord(msg) || !isUsableMessageId(msg.id))
|
|
4232
|
-
return; // malformed envelope (SPEC sec 5) — live is at-most-once: drop
|
|
4233
|
-
if (!isFromSubjectSender(msg, parsed))
|
|
4234
|
-
return; // spoof/malformed/old-shape-alias — drop (at-most-once)
|
|
4235
|
-
if (msg.from.id === this.card.id)
|
|
4236
|
-
return; // our own echo
|
|
4237
|
-
const delivery = { ack: () => { }, nak: () => { }, durable: false }; // live = at-most-once, not acked
|
|
4238
|
-
this.emit("message", authenticatedMessage(msg, parsed), delivery, {
|
|
4239
|
-
historical: false,
|
|
4240
|
-
kind: kindFromParsed(parsed.kind),
|
|
4241
|
-
});
|
|
4242
|
-
},
|
|
4243
|
-
});
|
|
4244
|
-
this.chatSubs.set(channel, sub);
|
|
4245
|
-
}
|
|
4246
|
-
/** Close a channel's core subscription (manager-free leave). */
|
|
4247
|
-
unsubscribeChat(channel) {
|
|
4248
|
-
const sub = this.chatSubs.get(channel);
|
|
4249
|
-
if (sub) {
|
|
4250
|
-
try {
|
|
4251
|
-
sub.unsubscribe();
|
|
4252
|
-
}
|
|
4253
|
-
catch {
|
|
4254
|
-
/* closing with the connection */
|
|
4255
|
-
}
|
|
4256
|
-
this.chatSubs.delete(channel);
|
|
4257
|
-
}
|
|
4258
|
-
this.chatSubDenied.delete(channel);
|
|
4259
|
-
}
|
|
4260
|
-
/** Confirm a just-opened core subscription was accepted by the broker. A `sub.allow` violation is
|
|
4261
|
-
* async in NATS, so flush (round-trips the SUB) then settle briefly to let the refusal land — a
|
|
4262
|
-
* denied subscribe must not read as a successful join (SPEC conformance #13). */
|
|
4263
|
-
async confirmChatSub() {
|
|
4264
|
-
if (!this.nc)
|
|
4265
|
-
throw new Error("connection not established");
|
|
4266
|
-
// flush() is the deterministic boundary: the broker's -ERR for an out-of-ACL SUB arrives BEFORE the
|
|
4267
|
-
// PONG, so once flush resolves the subscribe callback has already recorded any denial. A flush
|
|
4268
|
-
// FAILURE means the connection drained/closed mid-join — we have no confirmation, so let it throw
|
|
4269
|
-
// (joinChannel fails closed) instead of swallowing it and continuing as if confirmed.
|
|
4270
|
-
await this.nc.flush();
|
|
4271
|
-
await new Promise((r) => setTimeout(r, 50));
|
|
4272
|
-
}
|
|
4273
|
-
/** The durable's info (rebind) or null (fresh — 404). Gates create/backfill to the join event
|
|
4274
|
-
* and exposes the current `filter_subjects` for restart reconciliation. */
|
|
4275
|
-
async consumerInfo(stream, durable) {
|
|
4276
|
-
if (!this.jsm)
|
|
4277
|
-
throw new Error(this.notLiveMsg());
|
|
4278
|
-
try {
|
|
4279
|
-
return await this.jsm.consumers.info(stream, durable);
|
|
4280
|
-
}
|
|
4281
|
-
catch (e) {
|
|
4282
|
-
if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
|
|
4283
|
-
return null; // structured absence — fresh durable
|
|
4284
|
-
throw e;
|
|
4285
|
-
}
|
|
4286
|
-
}
|
|
4287
|
-
/** Delete one named consumer, swallowing ONLY structured consumer/stream absence. */
|
|
4288
|
-
async deleteConsumerIfPresent(stream, durable) {
|
|
4289
|
-
if (!this.jsm)
|
|
4290
|
-
throw new Error(this.notLiveMsg());
|
|
4291
|
-
try {
|
|
4292
|
-
await this.jsm.consumers.delete(stream, durable);
|
|
4293
|
-
}
|
|
4294
|
-
catch (e) {
|
|
4295
|
-
if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound, JetStreamApiCodes.StreamNotFound))
|
|
4296
|
-
throw e;
|
|
4297
|
-
}
|
|
2946
|
+
/** The durable consumers of this endpoint. Every dependency is read when it is used, so a field
|
|
2947
|
+
* assigned after construction is seen. */
|
|
2948
|
+
consumers = new EndpointConsumers({
|
|
2949
|
+
nc: () => this.nc,
|
|
2950
|
+
jsm: () => this.jsm,
|
|
2951
|
+
setJsm: (value) => { this.jsm = value; },
|
|
2952
|
+
js: () => this.js,
|
|
2953
|
+
space: () => this.space,
|
|
2954
|
+
owner: () => this.owner,
|
|
2955
|
+
actor: () => this.actor,
|
|
2956
|
+
card: () => this.card,
|
|
2957
|
+
authed: () => this.authed,
|
|
2958
|
+
channels: () => this.channels,
|
|
2959
|
+
ackWaitMs: () => this.ackWaitMs,
|
|
2960
|
+
inactiveThresholdMs: () => this.inactiveThresholdMs,
|
|
2961
|
+
firstConnect: () => this.firstConnect,
|
|
2962
|
+
setFirstConnect: (value) => { this.firstConnect = value; },
|
|
2963
|
+
backfillFloor: () => this.backfillFloor,
|
|
2964
|
+
streamMsgs: () => this.streamMsgs,
|
|
2965
|
+
taskEpoch: () => this.taskEpoch,
|
|
2966
|
+
setTaskEpoch: (value) => { this.taskEpoch = value; },
|
|
2967
|
+
stopped: () => this.stopped,
|
|
2968
|
+
dmOnlyInFlight: () => this.dmOnlyInFlight,
|
|
2969
|
+
setDmOnlyInFlight: (value) => { this.dmOnlyInFlight = value; },
|
|
2970
|
+
chatSubs: () => this.chatSubs,
|
|
2971
|
+
chatSubDenied: () => this.chatSubDenied,
|
|
2972
|
+
chatSubRefusals: () => this.chatSubRefusals,
|
|
2973
|
+
plane3Channels: () => this.plane3Channels,
|
|
2974
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
2975
|
+
requireLifecycleUid: (what) => this.requireLifecycleUid(what),
|
|
2976
|
+
ensureDmDurable: (owner, actor, lifecycleUid, opts) => this.provisioning.ensureDmDurable(owner, actor, lifecycleUid, opts),
|
|
2977
|
+
pumpDlv: () => this.deliveryPlane.pumpDlv(),
|
|
2978
|
+
armJoin: (channels) => this.armJoin(channels),
|
|
2979
|
+
backfillArmed: (frontiers, floorSeq) => this.backfillArmed(frontiers, floorSeq),
|
|
2980
|
+
armBootDurableMemberships: () => this.bootMemberships.armBootDurableMemberships(),
|
|
2981
|
+
takeAnycast: (role, seq, id, count, epoch, nak) => this.takeAnycast(role, seq, id, count, epoch, nak),
|
|
2982
|
+
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
2983
|
+
closeRefusedMembership: (channel, generation) => this.durable.closeRefusedMembership(channel, generation),
|
|
2984
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
2985
|
+
});
|
|
2986
|
+
/** Forwards to the consumers component's {@link EndpointConsumers.manager}. */
|
|
2987
|
+
manager() {
|
|
2988
|
+
return this.consumers.manager();
|
|
4298
2989
|
}
|
|
4299
2990
|
/** Current frontier (last sequence) of the chat stream — a channel's join watermark, the
|
|
4300
2991
|
* focus-watermark a connector captures on entering `focus` (recall reads ambient after it), and
|
|
@@ -4307,22 +2998,14 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4307
2998
|
}
|
|
4308
2999
|
/** Whether this connection's account has a JetStream stream of this name. A missing stream is
|
|
4309
3000
|
* false. Any other failure is thrown, so a caller does not treat an unread broker as absence. */
|
|
4310
|
-
|
|
4311
|
-
|
|
4312
|
-
await (await this.manager()).streams.info(name);
|
|
4313
|
-
return true;
|
|
4314
|
-
}
|
|
4315
|
-
catch (e) {
|
|
4316
|
-
if (isStreamNotFound(e))
|
|
4317
|
-
return false;
|
|
4318
|
-
throw e;
|
|
4319
|
-
}
|
|
3001
|
+
streamRecorded(name) {
|
|
3002
|
+
return this.consumers.streamRecorded(name);
|
|
4320
3003
|
}
|
|
4321
3004
|
/** The bytes this connection's account holds in JetStream file storage, as `$JS.API.INFO`
|
|
4322
3005
|
* reports them. A broker's `max_file_store` cap counts every account it serves, so this is a
|
|
4323
3006
|
* lower bound on what that cap measures. */
|
|
4324
|
-
|
|
4325
|
-
return
|
|
3007
|
+
storedBytes() {
|
|
3008
|
+
return this.consumers.storedBytes();
|
|
4326
3009
|
}
|
|
4327
3010
|
/** Phase 1 of a join: capture each channel's current CHAT stream frontier before opening the
|
|
4328
3011
|
* core subscription. The frontier bounds the {@link backfillArmed} read (`seq <= frontier`),
|
|
@@ -4444,18 +3127,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4444
3127
|
async publishPresence() {
|
|
4445
3128
|
return this.presence.publishPresence();
|
|
4446
3129
|
}
|
|
4447
|
-
/** #1356: drop the presence-refusal record when the connection that OBSERVED those refusals goes
|
|
4448
|
-
* away. "This bucket is refusing writes" is a claim about a specific broker connection; once that
|
|
4449
|
-
* connection is torn down or rebuilt the claim has no remaining basis, and a later failure on a
|
|
4450
|
-
* fresh connection must establish it again from its own evidence.
|
|
4451
|
-
*
|
|
4452
|
-
* Cleared at the SOURCE rather than guarded at each reader, because a guard protects one consumer
|
|
4453
|
-
* and clearing protects every consumer, including ones not yet written. Measured: without this, a
|
|
4454
|
-
* failed bind against an unreachable server still reported the presence-refusal sentence while the
|
|
4455
|
-
* endpoint's own `connectionIssue` already said "connection refused". */
|
|
4456
|
-
clearPresenceWriteFailure() {
|
|
4457
|
-
return this.presence.clearPresenceWriteFailure();
|
|
4458
|
-
}
|
|
4459
3130
|
/** #1356: the presence bucket has been refusing writes since this time, or `undefined` when the
|
|
4460
3131
|
* last publish succeeded. Cleared by a successful write that is the latest evidence on its
|
|
4461
3132
|
* epoch (#1461), so a survived blip reads as healthy and only a SUSTAINED failure carries a
|
|
@@ -4482,14 +3153,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4482
3153
|
async readPresenceOnce() {
|
|
4483
3154
|
return this.presence.readPresenceOnce();
|
|
4484
3155
|
}
|
|
4485
|
-
/** Forwards to the channel discovery component's {@link EndpointChannelDiscovery.startChannelWatch}. */
|
|
4486
|
-
async startChannelWatch() {
|
|
4487
|
-
return this.channelDiscovery.startChannelWatch();
|
|
4488
|
-
}
|
|
4489
|
-
/** Forwards to the channel discovery component's {@link EndpointChannelDiscovery.readChannelsOnce}. */
|
|
4490
|
-
async readChannelsOnce() {
|
|
4491
|
-
return this.channelDiscovery.readChannelsOnce();
|
|
4492
|
-
}
|
|
4493
3156
|
sweep() {
|
|
4494
3157
|
return this.presence.sweep();
|
|
4495
3158
|
}
|