@peerbit/pubsub 5.4.9 → 5.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/fanout-tree.d.ts +11 -1
- package/dist/src/fanout-tree.d.ts.map +1 -1
- package/dist/src/fanout-tree.js +228 -38
- package/dist/src/fanout-tree.js.map +1 -1
- package/dist/src/index.d.ts +11 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +612 -127
- package/dist/src/index.js.map +1 -1
- package/package.json +7 -7
- package/src/fanout-tree.ts +301 -42
- package/src/index.ts +857 -177
package/dist/src/index.js
CHANGED
|
@@ -144,6 +144,8 @@ const TOPIC_ROOT_QUERY_ROUND_TIMEOUT_MS = 3_000;
|
|
|
144
144
|
const TOPIC_ROOT_HOST_SCAN_TIMEOUT_MS = DEFAULT_TOPIC_ROOT_QUERY_TIMEOUT_MS;
|
|
145
145
|
const TOPIC_ROOT_HOST_SCAN_CONCURRENCY = 8;
|
|
146
146
|
const DIRECT_SHARD_ROOT_CONFIRM_TIMEOUT_MS = 2_000;
|
|
147
|
+
const RELAY_ROOT_PREFLIGHT_TIMEOUT_MS = 2_000;
|
|
148
|
+
const RELAY_ROOT_PREFLIGHT_REUSE_MS = 2_000;
|
|
147
149
|
const DEFAULT_PUBSUB_FANOUT_CHANNEL_OPTIONS = {
|
|
148
150
|
msgRate: 30,
|
|
149
151
|
msgSize: 1024,
|
|
@@ -225,9 +227,9 @@ export class TopicControlPlane extends DirectStream {
|
|
|
225
227
|
hostOwnedShardRootsInFlight;
|
|
226
228
|
hostOwnedShardRootsDirty = false;
|
|
227
229
|
signedTopicRootCandidateClaims = new Map();
|
|
228
|
-
|
|
229
|
-
//
|
|
230
|
-
//
|
|
230
|
+
relayRootProbeCohort;
|
|
231
|
+
// Locally observed unreachability makes only that origin's current lease
|
|
232
|
+
// ineffective locally. This is not a component-wide withdrawal. Keeping the verified
|
|
231
233
|
// bytes + replay floor prevents a relay from resurrecting the departed lease;
|
|
232
234
|
// the map is bounded by the same retained-origin cap as the claim map.
|
|
233
235
|
suppressedDepartedTopicRootCandidateClaims = new Map();
|
|
@@ -245,6 +247,34 @@ export class TopicControlPlane extends DirectStream {
|
|
|
245
247
|
refilledAt: Date.now(),
|
|
246
248
|
};
|
|
247
249
|
_onFanoutPeerUnreachable;
|
|
250
|
+
onFanoutAttached = (ev) => {
|
|
251
|
+
const st = this.fanoutChannels.get(ev.detail.topic);
|
|
252
|
+
if (!st || st.root !== ev.detail.root)
|
|
253
|
+
return;
|
|
254
|
+
// A successful publish before attachment may have had no listeners.
|
|
255
|
+
// Coalesce attachment bursts, retaining one trailing pass during a send.
|
|
256
|
+
st.announceDirty = true;
|
|
257
|
+
if (st.announceTask)
|
|
258
|
+
return;
|
|
259
|
+
const revision = this.topicControlPlaneLifecycleRevision;
|
|
260
|
+
st.announceTask = Promise.resolve()
|
|
261
|
+
.then(async () => {
|
|
262
|
+
await st.join;
|
|
263
|
+
while (st.announceDirty &&
|
|
264
|
+
this.isFanoutChannelCurrent(ev.detail.topic, st.channel, revision)) {
|
|
265
|
+
st.announceDirty = false;
|
|
266
|
+
await this.announceShardSubscriptions(ev.detail.topic, revision);
|
|
267
|
+
}
|
|
268
|
+
})
|
|
269
|
+
.catch(logErrorIfStarted)
|
|
270
|
+
.finally(() => {
|
|
271
|
+
st.announceTask = undefined;
|
|
272
|
+
if (st.announceDirty &&
|
|
273
|
+
this.isFanoutChannelCurrent(ev.detail.topic, st.channel, revision)) {
|
|
274
|
+
this.onFanoutAttached(ev);
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
};
|
|
248
278
|
fanoutChannels = new Map();
|
|
249
279
|
ensureFanoutChannelInFlight = new Map();
|
|
250
280
|
pendingTopicRootQueries = new Map();
|
|
@@ -387,6 +417,8 @@ export class TopicControlPlane extends DirectStream {
|
|
|
387
417
|
this._onFanoutPeerUnreachable =
|
|
388
418
|
this._onFanoutPeerUnreachable || this.onFanoutPeerUnreachable.bind(this);
|
|
389
419
|
await this.fanout.addEventListener("fanout:peer-unreachable", this._onFanoutPeerUnreachable);
|
|
420
|
+
this.fanout.addEventListener("fanout:joined", this.onFanoutAttached);
|
|
421
|
+
this.fanout.addEventListener("fanout:child-joined", this.onFanoutAttached);
|
|
390
422
|
await super.start();
|
|
391
423
|
if (this.autoTopicRootCandidates) {
|
|
392
424
|
let retryDelayMs = 0;
|
|
@@ -427,6 +459,8 @@ export class TopicControlPlane extends DirectStream {
|
|
|
427
459
|
if (this._onFanoutPeerUnreachable) {
|
|
428
460
|
this.fanout.removeEventListener("fanout:peer-unreachable", this._onFanoutPeerUnreachable);
|
|
429
461
|
}
|
|
462
|
+
this.fanout.removeEventListener("fanout:joined", this.onFanoutAttached);
|
|
463
|
+
this.fanout.removeEventListener("fanout:child-joined", this.onFanoutAttached);
|
|
430
464
|
for (const st of this.fanoutChannels.values()) {
|
|
431
465
|
this.clearFanoutIdleClose(st);
|
|
432
466
|
try {
|
|
@@ -491,22 +525,23 @@ export class TopicControlPlane extends DirectStream {
|
|
|
491
525
|
this.rebuildAutoTopicRootCandidatesFromClaims(BigInt(Date.now()), {
|
|
492
526
|
immediate: hadSignedPeer !== hasSignedPeer,
|
|
493
527
|
});
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
void this.sendAutoTopicRootCandidates([peer]).catch(() => { });
|
|
504
|
-
}
|
|
505
|
-
if (existingPeer !== peer) {
|
|
506
|
-
peer.addEventListener("stream:outbound", sendWhenOutboundReady);
|
|
507
|
-
peer.addEventListener("close", () => peer.removeEventListener("stream:outbound", sendWhenOutboundReady), { once: true });
|
|
508
|
-
}
|
|
528
|
+
}
|
|
529
|
+
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
530
|
+
const sendWhenOutboundReady = () => {
|
|
531
|
+
if (!this.isTopicControlPlaneActive(lifecycleRevision) ||
|
|
532
|
+
this.peers.get(publicKey.hashcode()) !== peer)
|
|
533
|
+
return;
|
|
534
|
+
if (this.autoTopicRootCandidates &&
|
|
535
|
+
peer.protocol === TOPIC_CONTROL_PLANE_PROTOCOL_V2_1) {
|
|
536
|
+
void this.sendAutoTopicRootCandidates([peer]).catch(() => { });
|
|
509
537
|
}
|
|
538
|
+
void this.announceDirectSubscriptions([...this.subscriptions.keys()], [peer]).catch(logErrorIfStarted);
|
|
539
|
+
};
|
|
540
|
+
if (peer.isWritable)
|
|
541
|
+
sendWhenOutboundReady();
|
|
542
|
+
if (existingPeer !== peer) {
|
|
543
|
+
peer.addEventListener("stream:outbound", sendWhenOutboundReady);
|
|
544
|
+
peer.addEventListener("close", () => peer.removeEventListener("stream:outbound", sendWhenOutboundReady), { once: true });
|
|
510
545
|
}
|
|
511
546
|
return peer;
|
|
512
547
|
}
|
|
@@ -524,15 +559,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
524
559
|
protocol === TOPIC_CONTROL_PLANE_PROTOCOL_V2_1 &&
|
|
525
560
|
this.signedTopicRootCandidateClaims.has(hash)) {
|
|
526
561
|
const claim = this.signedTopicRootCandidateClaims.get(hash);
|
|
527
|
-
|
|
528
|
-
suppressedDepartedOrigin =
|
|
529
|
-
!suppressed ||
|
|
530
|
-
suppressed.timestamp !== claim.timestamp ||
|
|
531
|
-
!bytesEqual(suppressed.bytes, claim.bytes);
|
|
532
|
-
this.suppressedDepartedTopicRootCandidateClaims.set(hash, {
|
|
533
|
-
bytes: claim.bytes,
|
|
534
|
-
timestamp: claim.timestamp,
|
|
535
|
-
});
|
|
562
|
+
suppressedDepartedOrigin = this.suppressTopicRootCandidateClaim(hash, claim);
|
|
536
563
|
}
|
|
537
564
|
if (this.autoTopicRootCandidates &&
|
|
538
565
|
(protocol === TOPIC_CONTROL_PLANE_PROTOCOL_V2_0 ||
|
|
@@ -576,6 +603,8 @@ export class TopicControlPlane extends DirectStream {
|
|
|
576
603
|
}
|
|
577
604
|
}
|
|
578
605
|
clearSignedTopicRootCandidateState() {
|
|
606
|
+
this.relayRootProbeCohort?.controller.abort(new AbortError("topic root candidate recovery stopped"));
|
|
607
|
+
this.relayRootProbeCohort = undefined;
|
|
579
608
|
this.clearTopicRootCandidateClaimTimer();
|
|
580
609
|
this.signedTopicRootCandidateClaims.clear();
|
|
581
610
|
this.suppressedDepartedTopicRootCandidateClaims.clear();
|
|
@@ -862,6 +891,264 @@ export class TopicControlPlane extends DirectStream {
|
|
|
862
891
|
return (suppressed?.timestamp === claim.timestamp &&
|
|
863
892
|
bytesEqual(suppressed.bytes, claim.bytes));
|
|
864
893
|
}
|
|
894
|
+
suppressTopicRootCandidateClaim(origin, claim) {
|
|
895
|
+
if (!this.autoTopicRootCandidates ||
|
|
896
|
+
origin === this.publicKeyHash ||
|
|
897
|
+
this.peers.has(origin) ||
|
|
898
|
+
this.signedTopicRootCandidateClaims.get(origin) !== claim ||
|
|
899
|
+
this.isDepartedTopicRootCandidateClaimSuppressed(origin, claim))
|
|
900
|
+
return false;
|
|
901
|
+
this.suppressedDepartedTopicRootCandidateClaims.set(origin, {
|
|
902
|
+
bytes: claim.bytes,
|
|
903
|
+
timestamp: claim.timestamp,
|
|
904
|
+
});
|
|
905
|
+
return true;
|
|
906
|
+
}
|
|
907
|
+
topicRootRouteObservations(origin) {
|
|
908
|
+
return this.routes
|
|
909
|
+
.getRouteHints(this.publicKeyHash, origin)
|
|
910
|
+
.map((hint) => JSON.stringify([hint.nextHop, hint.session, hint.updatedAt]));
|
|
911
|
+
}
|
|
912
|
+
createRelayRootProbeCohort(claims) {
|
|
913
|
+
const previous = this.relayRootProbeCohort?.results;
|
|
914
|
+
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
915
|
+
const controller = new AbortController();
|
|
916
|
+
const users = new Set();
|
|
917
|
+
const deadline = new AbortError("relay root reachability deadline reached");
|
|
918
|
+
const timer = setTimeout(() => controller.abort(deadline), RELAY_ROOT_PREFLIGHT_TIMEOUT_MS);
|
|
919
|
+
timer.unref?.();
|
|
920
|
+
let resolveRetry;
|
|
921
|
+
const retryBoundary = new Promise((resolve) => {
|
|
922
|
+
resolveRetry = resolve;
|
|
923
|
+
});
|
|
924
|
+
const retryTimer = setTimeout(resolveRetry, RELAY_ROOT_PREFLIGHT_TIMEOUT_MS / 2);
|
|
925
|
+
retryTimer.unref?.();
|
|
926
|
+
const captured = claims.map(([origin, claim]) => ({
|
|
927
|
+
origin,
|
|
928
|
+
claim,
|
|
929
|
+
routes: this.topicRootRouteObservations(origin),
|
|
930
|
+
}));
|
|
931
|
+
const cohort = {
|
|
932
|
+
controller,
|
|
933
|
+
users,
|
|
934
|
+
lifecycleRevision,
|
|
935
|
+
promise: Promise.resolve()
|
|
936
|
+
.then(async () => {
|
|
937
|
+
const outcomes = await Promise.all(captured.map(async ({ origin, claim, routes }) => {
|
|
938
|
+
const cached = previous?.get(origin);
|
|
939
|
+
if (cached?.claim === claim &&
|
|
940
|
+
cached.status === "alive" &&
|
|
941
|
+
performance.now() - cached.observedAt <
|
|
942
|
+
RELAY_ROOT_PREFLIGHT_REUSE_MS)
|
|
943
|
+
return [origin, { ...cached, routes }];
|
|
944
|
+
const winner = new AbortController();
|
|
945
|
+
const signals = linkAbortSignals([
|
|
946
|
+
controller.signal,
|
|
947
|
+
winner.signal,
|
|
948
|
+
]);
|
|
949
|
+
let alive = false;
|
|
950
|
+
let resolveAlive;
|
|
951
|
+
const firstSuccess = new Promise((resolve) => {
|
|
952
|
+
resolveAlive = resolve;
|
|
953
|
+
});
|
|
954
|
+
const probe = async (retry) => {
|
|
955
|
+
const signal = signals.signal;
|
|
956
|
+
let submitted = false;
|
|
957
|
+
try {
|
|
958
|
+
if (retry)
|
|
959
|
+
await withAbort(retryBoundary, signal);
|
|
960
|
+
throwIfAborted(signal);
|
|
961
|
+
if (alive)
|
|
962
|
+
return "alive";
|
|
963
|
+
const message = await withAbort(this.createMessage(undefined, {
|
|
964
|
+
mode: new AcknowledgeDelivery({
|
|
965
|
+
to: [origin],
|
|
966
|
+
redundancy: 1,
|
|
967
|
+
}),
|
|
968
|
+
priority: 1,
|
|
969
|
+
responsePriority: 1,
|
|
970
|
+
}), signal);
|
|
971
|
+
// Signers may ignore cancellation. Fence the send continuation.
|
|
972
|
+
throwIfAborted(signal);
|
|
973
|
+
if (alive)
|
|
974
|
+
return "alive";
|
|
975
|
+
if (!this.isTopicControlPlaneActive(lifecycleRevision) ||
|
|
976
|
+
![...users].some((current) => current())) {
|
|
977
|
+
return "unknown";
|
|
978
|
+
}
|
|
979
|
+
// No transport neighbor is local readiness, not proof of origin death.
|
|
980
|
+
if (this.peers.size === 0)
|
|
981
|
+
return "unknown";
|
|
982
|
+
submitted = true;
|
|
983
|
+
await withAbort(this.publishMessage(this.publicKey, message, undefined, undefined, signal), signal);
|
|
984
|
+
throwIfAborted(signal);
|
|
985
|
+
alive = true;
|
|
986
|
+
resolveAlive("alive");
|
|
987
|
+
return "alive";
|
|
988
|
+
}
|
|
989
|
+
catch (error) {
|
|
990
|
+
if (submitted &&
|
|
991
|
+
(error === deadline || error instanceof DeliveryError))
|
|
992
|
+
return "failed";
|
|
993
|
+
else if (!signal.aborted)
|
|
994
|
+
throw error;
|
|
995
|
+
return "unknown";
|
|
996
|
+
}
|
|
997
|
+
};
|
|
998
|
+
// Hedge once at the shared midpoint without discarding the first
|
|
999
|
+
// authenticated ACK wait: a 1.5s first reply is still timely.
|
|
1000
|
+
let status;
|
|
1001
|
+
try {
|
|
1002
|
+
status = await Promise.race([
|
|
1003
|
+
firstSuccess,
|
|
1004
|
+
Promise.all([probe(false), probe(true)]).then((results) => results.includes("unknown")
|
|
1005
|
+
? "unknown"
|
|
1006
|
+
: "failed"),
|
|
1007
|
+
]);
|
|
1008
|
+
}
|
|
1009
|
+
finally {
|
|
1010
|
+
winner.abort(new AbortError("origin probe settled"));
|
|
1011
|
+
signals.clear();
|
|
1012
|
+
}
|
|
1013
|
+
return [
|
|
1014
|
+
origin,
|
|
1015
|
+
{ claim, routes, status, observedAt: performance.now() },
|
|
1016
|
+
];
|
|
1017
|
+
}));
|
|
1018
|
+
cohort.results = new Map(outcomes);
|
|
1019
|
+
return cohort.results;
|
|
1020
|
+
})
|
|
1021
|
+
.finally(() => {
|
|
1022
|
+
clearTimeout(timer);
|
|
1023
|
+
clearTimeout(retryTimer);
|
|
1024
|
+
controller.abort(new AbortError("relay root probes settled"));
|
|
1025
|
+
}),
|
|
1026
|
+
};
|
|
1027
|
+
this.relayRootProbeCohort = cohort;
|
|
1028
|
+
return cohort;
|
|
1029
|
+
}
|
|
1030
|
+
async preflightRelayRootCandidates(shardTopic, attempt, signal, isCurrent, localPolicyOnly = false) {
|
|
1031
|
+
throwIfAborted(signal);
|
|
1032
|
+
const revision = this.topicControlPlaneLifecycleRevision;
|
|
1033
|
+
const current = () => isCurrent() && this.isTopicControlPlaneActive(revision);
|
|
1034
|
+
if (!current())
|
|
1035
|
+
throw new AbortError("obsolete relay root recovery");
|
|
1036
|
+
if (!this.autoTopicRootCandidates ||
|
|
1037
|
+
this.maybeDisableAutoTopicRootCandidatesIfExternallyConfigured())
|
|
1038
|
+
return false;
|
|
1039
|
+
const claims = [...this.signedTopicRootCandidateClaims.entries()]
|
|
1040
|
+
.filter(([origin, claim]) => origin !== this.publicKeyHash &&
|
|
1041
|
+
!this.peers.has(origin) &&
|
|
1042
|
+
!this.isDepartedTopicRootCandidateClaimSuppressed(origin, claim))
|
|
1043
|
+
.slice(0, TOPIC_ROOT_CANDIDATES_MAX);
|
|
1044
|
+
if (claims.length === 0)
|
|
1045
|
+
return false;
|
|
1046
|
+
const shared = this.relayRootProbeCohort;
|
|
1047
|
+
const cohort = (attempt.cohort ??=
|
|
1048
|
+
shared &&
|
|
1049
|
+
(shared.results
|
|
1050
|
+
? shared.users.size > 0
|
|
1051
|
+
: !shared.controller.signal.aborted) &&
|
|
1052
|
+
shared.lifecycleRevision === revision
|
|
1053
|
+
? shared
|
|
1054
|
+
: this.createRelayRootProbeCohort(claims));
|
|
1055
|
+
cohort.users.add(current);
|
|
1056
|
+
let held = true;
|
|
1057
|
+
const release = () => {
|
|
1058
|
+
if (!held)
|
|
1059
|
+
return;
|
|
1060
|
+
held = false;
|
|
1061
|
+
cohort.users.delete(current);
|
|
1062
|
+
if (cohort.users.size === 0 && !cohort.results) {
|
|
1063
|
+
cohort.controller.abort(new AbortError("relay root recovery cancelled"));
|
|
1064
|
+
}
|
|
1065
|
+
};
|
|
1066
|
+
signal?.addEventListener("abort", release, { once: true });
|
|
1067
|
+
try {
|
|
1068
|
+
const outcomes = await withAbort(cohort.promise, signal);
|
|
1069
|
+
throwIfAborted(signal);
|
|
1070
|
+
if (!current())
|
|
1071
|
+
throw new AbortError("obsolete relay root recovery");
|
|
1072
|
+
// Policy setters do not share the candidate generation. Re-read configured
|
|
1073
|
+
// roots/resolvers/trackers after observation; a stalled policy is unknown,
|
|
1074
|
+
// never evidence authorizing local suppression.
|
|
1075
|
+
attempt.policyDeadline ??=
|
|
1076
|
+
performance.now() + RELAY_ROOT_PREFLIGHT_TIMEOUT_MS;
|
|
1077
|
+
return await this.withRelayRootPolicyDeadline(attempt, signal, async (policySignal) => {
|
|
1078
|
+
const configured = await withAbort(localPolicyOnly
|
|
1079
|
+
? this.topicRootControlPlane.resolveLocalTopicRoot(shardTopic, {
|
|
1080
|
+
signal: policySignal,
|
|
1081
|
+
})
|
|
1082
|
+
: this.topicRootControlPlane.resolveTrackedTopicRoot(shardTopic, {
|
|
1083
|
+
signal: policySignal,
|
|
1084
|
+
}), policySignal);
|
|
1085
|
+
throwIfAborted(policySignal);
|
|
1086
|
+
if (!current())
|
|
1087
|
+
throw new AbortError("obsolete relay root recovery");
|
|
1088
|
+
this.shardRootCache.delete(shardTopic);
|
|
1089
|
+
if (configured ||
|
|
1090
|
+
this.topicRootControlPlane.getTopicRoot(shardTopic) ||
|
|
1091
|
+
!this.autoTopicRootCandidates ||
|
|
1092
|
+
this.maybeDisableAutoTopicRootCandidatesIfExternallyConfigured())
|
|
1093
|
+
return true;
|
|
1094
|
+
let changed = false;
|
|
1095
|
+
let unknown = false;
|
|
1096
|
+
for (const [origin, claim] of claims) {
|
|
1097
|
+
if (this.peers.has(origin) ||
|
|
1098
|
+
this.isDepartedTopicRootCandidateClaimSuppressed(origin, claim))
|
|
1099
|
+
continue;
|
|
1100
|
+
const observed = outcomes.get(origin);
|
|
1101
|
+
if (this.signedTopicRootCandidateClaims.get(origin) !== claim ||
|
|
1102
|
+
observed?.claim !== claim) {
|
|
1103
|
+
unknown = true;
|
|
1104
|
+
continue;
|
|
1105
|
+
}
|
|
1106
|
+
const freshRoute = this.routes.isReachable(this.publicKeyHash, origin) &&
|
|
1107
|
+
this.topicRootRouteObservations(origin).some((route) => !observed.routes.includes(route));
|
|
1108
|
+
if (freshRoute || observed.status === "alive")
|
|
1109
|
+
continue;
|
|
1110
|
+
if (observed.status === "unknown")
|
|
1111
|
+
unknown = true;
|
|
1112
|
+
else
|
|
1113
|
+
changed =
|
|
1114
|
+
this.suppressTopicRootCandidateClaim(origin, claim) || changed;
|
|
1115
|
+
}
|
|
1116
|
+
if (changed)
|
|
1117
|
+
this.rebuildAutoTopicRootCandidatesFromClaims(BigInt(Date.now()), {
|
|
1118
|
+
immediate: true,
|
|
1119
|
+
});
|
|
1120
|
+
if (unknown)
|
|
1121
|
+
throw new DeliveryError("Relay root reachability could not be established");
|
|
1122
|
+
return true;
|
|
1123
|
+
});
|
|
1124
|
+
}
|
|
1125
|
+
finally {
|
|
1126
|
+
signal?.removeEventListener("abort", release);
|
|
1127
|
+
release();
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
async withRelayRootPolicyDeadline(attempt, signal, resolve) {
|
|
1131
|
+
// Ordinary initial resolution retains its existing contract. Only work
|
|
1132
|
+
// following a probe shares the one additional policy budget, including retries.
|
|
1133
|
+
if (attempt.policyDeadline === undefined)
|
|
1134
|
+
return resolve(signal);
|
|
1135
|
+
throwIfAborted(signal);
|
|
1136
|
+
const remaining = attempt.policyDeadline - performance.now();
|
|
1137
|
+
const deadline = new AbortError("relay root policy recheck deadline reached");
|
|
1138
|
+
if (remaining <= 0)
|
|
1139
|
+
throw deadline;
|
|
1140
|
+
const controller = new AbortController();
|
|
1141
|
+
const timer = setTimeout(() => controller.abort(deadline), remaining);
|
|
1142
|
+
timer.unref?.();
|
|
1143
|
+
const signals = linkAbortSignals([signal, controller.signal]);
|
|
1144
|
+
try {
|
|
1145
|
+
return await withAbort(resolve(signals.signal), signals.signal);
|
|
1146
|
+
}
|
|
1147
|
+
finally {
|
|
1148
|
+
clearTimeout(timer);
|
|
1149
|
+
signals.clear();
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
865
1152
|
canAdvanceTopicRootCandidateClaimReplayFloor(origin, timestamp) {
|
|
866
1153
|
const floor = this.topicRootCandidateClaimReplayFloors.get(origin);
|
|
867
1154
|
if (floor !== undefined) {
|
|
@@ -922,7 +1209,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
922
1209
|
this.topicRootCandidateClaimGlobalVerifyBudget.remaining -= 1;
|
|
923
1210
|
return true;
|
|
924
1211
|
}
|
|
925
|
-
async importSignedTopicRootCandidateClaim(rawClaim, outerPeerHash, expectedPeer) {
|
|
1212
|
+
async importSignedTopicRootCandidateClaim(rawClaim, outerPeerHash, expectedPeer, batchUpdate) {
|
|
926
1213
|
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
927
1214
|
if (rawClaim.byteLength === 0 ||
|
|
928
1215
|
rawClaim.byteLength > TOPIC_ROOT_CANDIDATE_CLAIM_MAX_BYTES) {
|
|
@@ -961,7 +1248,13 @@ export class TopicControlPlane extends DirectStream {
|
|
|
961
1248
|
if (acceptanceDeadline === undefined) {
|
|
962
1249
|
return false;
|
|
963
1250
|
}
|
|
964
|
-
|
|
1251
|
+
if (batchUpdate) {
|
|
1252
|
+
if (this.pruneExpiredTopicRootCandidateClaims(now))
|
|
1253
|
+
batchUpdate.immediate = true;
|
|
1254
|
+
}
|
|
1255
|
+
else {
|
|
1256
|
+
this.rebuildAutoTopicRootCandidatesFromClaims(now);
|
|
1257
|
+
}
|
|
965
1258
|
const retained = this.signedTopicRootCandidateClaims.get(origin);
|
|
966
1259
|
if (outerPeerHash === origin &&
|
|
967
1260
|
expectedPeer !== undefined &&
|
|
@@ -979,7 +1272,10 @@ export class TopicControlPlane extends DirectStream {
|
|
|
979
1272
|
// monotonic acceptance deadline. A current authenticated origin stream is
|
|
980
1273
|
// sufficient liveness evidence; do not advance the floor or lease.
|
|
981
1274
|
this.suppressedDepartedTopicRootCandidateClaims.delete(origin);
|
|
982
|
-
|
|
1275
|
+
if (batchUpdate)
|
|
1276
|
+
batchUpdate.immediate = true;
|
|
1277
|
+
else
|
|
1278
|
+
this.rebuildAutoTopicRootCandidatesFromClaims(now, { immediate: true });
|
|
983
1279
|
return false;
|
|
984
1280
|
}
|
|
985
1281
|
if (!this.canAdvanceTopicRootCandidateClaimReplayFloor(origin, timestamp)) {
|
|
@@ -1231,25 +1527,19 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1231
1527
|
}
|
|
1232
1528
|
// Ensure shard overlays are joined using the current root mapping (may
|
|
1233
1529
|
// migrate channels if roots changed), then re-announce subscriptions.
|
|
1234
|
-
await Promise.
|
|
1530
|
+
const results = await Promise.allSettled([...byShard.entries()].map(async ([shardTopic, userTopics]) => {
|
|
1235
1531
|
if (userTopics.length === 0)
|
|
1236
1532
|
return;
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
});
|
|
1242
|
-
const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
|
|
1243
|
-
mode: new AnyWhere(),
|
|
1244
|
-
priority: 1,
|
|
1245
|
-
skipRecipientValidation: true,
|
|
1533
|
+
const owners = userTopics.map((topic) => [topic, this.subscriptions.get(topic)]);
|
|
1534
|
+
await this.ensureFanoutChannel(shardTopic, {
|
|
1535
|
+
ephemeral: false,
|
|
1536
|
+
isCurrent: () => owners.some(([topic, owner]) => owner && this.subscriptions.get(topic) === owner),
|
|
1246
1537
|
});
|
|
1247
|
-
|
|
1248
|
-
if (!st)
|
|
1249
|
-
return;
|
|
1250
|
-
await st.channel.publish(toUint8Array(embedded.bytes()));
|
|
1251
|
-
this.touchFanoutChannel(shardTopic, st.channel, lifecycleRevision);
|
|
1538
|
+
await this.announceShardSubscriptions(shardTopic, lifecycleRevision);
|
|
1252
1539
|
}));
|
|
1540
|
+
const failure = results.find((result) => result.status === "rejected");
|
|
1541
|
+
if (failure?.status === "rejected")
|
|
1542
|
+
throw failure.reason;
|
|
1253
1543
|
}
|
|
1254
1544
|
isTrackedTopic(topic) {
|
|
1255
1545
|
return this.topics.has(topic);
|
|
@@ -1260,16 +1550,17 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1260
1550
|
untrackTopic(topic) {
|
|
1261
1551
|
const peers = this.topics.get(topic);
|
|
1262
1552
|
this.topics.delete(topic);
|
|
1263
|
-
|
|
1264
|
-
return;
|
|
1265
|
-
for (const peerHash of peers.keys()) {
|
|
1553
|
+
for (const peerHash of peers?.keys() ?? []) {
|
|
1266
1554
|
this.peerToTopic.get(peerHash)?.delete(topic);
|
|
1267
|
-
this.lastSubscriptionMessages.get(peerHash)?.delete(topic);
|
|
1268
1555
|
if (!this.peerToTopic.get(peerHash)?.size) {
|
|
1269
1556
|
this.peerToTopic.delete(peerHash);
|
|
1270
|
-
this.lastSubscriptionMessages.delete(peerHash);
|
|
1271
1557
|
}
|
|
1272
1558
|
}
|
|
1559
|
+
for (const [peerHash, messages] of this.lastSubscriptionMessages) {
|
|
1560
|
+
messages.delete(topic);
|
|
1561
|
+
if (messages.size === 0)
|
|
1562
|
+
this.lastSubscriptionMessages.delete(peerHash);
|
|
1563
|
+
}
|
|
1273
1564
|
}
|
|
1274
1565
|
initializePeer(publicKey) {
|
|
1275
1566
|
// Remote subscribers can be non-neighbours (learned via relayed topic control).
|
|
@@ -1292,10 +1583,26 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1292
1583
|
this.lastSubscriptionMessages.get(oldest)?.delete(topic);
|
|
1293
1584
|
if (!this.peerToTopic.get(oldest)?.size) {
|
|
1294
1585
|
this.peerToTopic.delete(oldest);
|
|
1586
|
+
}
|
|
1587
|
+
if (!this.lastSubscriptionMessages.get(oldest)?.size) {
|
|
1295
1588
|
this.lastSubscriptionMessages.delete(oldest);
|
|
1296
1589
|
}
|
|
1297
1590
|
}
|
|
1298
1591
|
}
|
|
1592
|
+
pruneUnsubscribedWatermarks(topic) {
|
|
1593
|
+
let remaining = this.subscriberCacheMaxEntries;
|
|
1594
|
+
for (const [peerHash, messages] of [
|
|
1595
|
+
...this.lastSubscriptionMessages,
|
|
1596
|
+
].reverse()) {
|
|
1597
|
+
if (!messages.has(topic) || this.topics.get(topic)?.has(peerHash))
|
|
1598
|
+
continue;
|
|
1599
|
+
if (remaining-- > 0)
|
|
1600
|
+
continue;
|
|
1601
|
+
messages.delete(topic);
|
|
1602
|
+
if (messages.size === 0)
|
|
1603
|
+
this.lastSubscriptionMessages.delete(peerHash);
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1299
1606
|
getSubscriptionOverlap(topics) {
|
|
1300
1607
|
const subscriptions = [];
|
|
1301
1608
|
if (topics) {
|
|
@@ -1566,20 +1873,27 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1566
1873
|
topic.startsWith(this.shardTopicPrefix) &&
|
|
1567
1874
|
deterministic &&
|
|
1568
1875
|
root !== deterministic) {
|
|
1569
|
-
return { root: deterministic, authoritative: false };
|
|
1876
|
+
return { root: deterministic, authoritative: false, autoCandidate: true };
|
|
1570
1877
|
}
|
|
1571
|
-
return {
|
|
1878
|
+
return {
|
|
1879
|
+
root,
|
|
1880
|
+
authoritative: true,
|
|
1881
|
+
autoCandidate: this.autoTopicRootCandidates &&
|
|
1882
|
+
topic.startsWith(this.shardTopicPrefix) &&
|
|
1883
|
+
root === deterministic,
|
|
1884
|
+
};
|
|
1572
1885
|
}
|
|
1573
1886
|
async resolveTopicRootState(topic, options) {
|
|
1574
1887
|
throwIfAborted(options?.signal);
|
|
1575
1888
|
const tracked = await this.topicRootControlPlane.resolveTrackedTopicRoot(topic, options);
|
|
1576
1889
|
if (tracked) {
|
|
1577
|
-
return { root: tracked, authoritative: true };
|
|
1890
|
+
return { root: tracked, authoritative: true, autoCandidate: false };
|
|
1578
1891
|
}
|
|
1579
1892
|
if (options?.queryConnectedPeers === false) {
|
|
1580
1893
|
return {
|
|
1581
1894
|
root: this.topicRootControlPlane.resolveDeterministicTopicRoot(topic),
|
|
1582
1895
|
authoritative: false,
|
|
1896
|
+
autoCandidate: this.autoTopicRootCandidates,
|
|
1583
1897
|
};
|
|
1584
1898
|
}
|
|
1585
1899
|
const deadlineController = new AbortController();
|
|
@@ -1641,6 +1955,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1641
1955
|
return {
|
|
1642
1956
|
root: this.topicRootControlPlane.resolveDeterministicTopicRoot(topic),
|
|
1643
1957
|
authoritative: false,
|
|
1958
|
+
autoCandidate: this.autoTopicRootCandidates,
|
|
1644
1959
|
};
|
|
1645
1960
|
}
|
|
1646
1961
|
async resolveTopicRoot(topic, options) {
|
|
@@ -1669,7 +1984,11 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1669
1984
|
? undefined
|
|
1670
1985
|
: this.shardRootCache.get(shardTopic);
|
|
1671
1986
|
if (cached && (cached.authoritative || !hasConnectedTrackers)) {
|
|
1672
|
-
return {
|
|
1987
|
+
return {
|
|
1988
|
+
root: cached.root,
|
|
1989
|
+
candidateGeneration,
|
|
1990
|
+
autoCandidate: cached.autoCandidate,
|
|
1991
|
+
};
|
|
1673
1992
|
}
|
|
1674
1993
|
const resolved = await this.resolveTopicRootState(shardTopic, options);
|
|
1675
1994
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
@@ -1685,7 +2004,11 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1685
2004
|
else {
|
|
1686
2005
|
this.shardRootCache.delete(shardTopic);
|
|
1687
2006
|
}
|
|
1688
|
-
return {
|
|
2007
|
+
return {
|
|
2008
|
+
root: resolved.root,
|
|
2009
|
+
candidateGeneration,
|
|
2010
|
+
autoCandidate: resolved.autoCandidate,
|
|
2011
|
+
};
|
|
1689
2012
|
}
|
|
1690
2013
|
}
|
|
1691
2014
|
getConnectedTopicRootTrackers() {
|
|
@@ -1709,6 +2032,11 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1709
2032
|
} while (true);
|
|
1710
2033
|
}
|
|
1711
2034
|
async sendDirectControlMessage(peer, pubsubMessage, signal) {
|
|
2035
|
+
const revision = this.topicControlPlaneLifecycleRevision;
|
|
2036
|
+
const isCurrent = () => this.isTopicControlPlaneActive(revision) &&
|
|
2037
|
+
this.peers.get(peer.publicKey.hashcode()) === peer;
|
|
2038
|
+
if (!isCurrent())
|
|
2039
|
+
return;
|
|
1712
2040
|
const embedded = await this.createMessage(this.encodePubSubMessage(pubsubMessage), {
|
|
1713
2041
|
mode: new SilentDelivery({
|
|
1714
2042
|
to: [peer.publicKey.hashcode()],
|
|
@@ -1717,7 +2045,9 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1717
2045
|
priority: 1,
|
|
1718
2046
|
skipRecipientValidation: true,
|
|
1719
2047
|
});
|
|
1720
|
-
|
|
2048
|
+
if (!isCurrent())
|
|
2049
|
+
return;
|
|
2050
|
+
await this.publishMessage(this.publicKey, embedded, [peer], undefined, signal ?? this.topicRootResolutionAbortController.signal);
|
|
1721
2051
|
}
|
|
1722
2052
|
resolvePendingTopicRootQuery(message, responderPeerHash) {
|
|
1723
2053
|
const pending = this.pendingTopicRootQueries.get(message.requestId);
|
|
@@ -1823,20 +2153,43 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1823
2153
|
this.shardRootCache.set(shardTopic, {
|
|
1824
2154
|
root: resolved.root,
|
|
1825
2155
|
authoritative: true,
|
|
2156
|
+
autoCandidate: resolved.autoCandidate,
|
|
1826
2157
|
});
|
|
1827
2158
|
}
|
|
1828
2159
|
return resolved.root;
|
|
1829
2160
|
}
|
|
1830
|
-
async resolveQueryableTopicRoot(topic) {
|
|
2161
|
+
async resolveQueryableTopicRoot(topic, isCurrent = () => true) {
|
|
1831
2162
|
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
1832
2163
|
const lifecycleSignal = this.topicRootResolutionAbortController.signal;
|
|
2164
|
+
const recovery = {};
|
|
1833
2165
|
return this.withTopicRootCandidateResolution(async ({ candidateGeneration, signal }) => {
|
|
1834
2166
|
throwIfAborted(signal);
|
|
1835
2167
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
1836
|
-
|
|
1837
|
-
|
|
2168
|
+
if (!isCurrent())
|
|
2169
|
+
throw new AbortError("obsolete topic root query");
|
|
2170
|
+
// Incoming queries retain canonical directory semantics: never ask
|
|
2171
|
+
// trackers here, since a tracker may itself query this peer.
|
|
2172
|
+
const resolveCanonical = () => this.withRelayRootPolicyDeadline(recovery, signal, async (policySignal) => {
|
|
2173
|
+
const local = await this.topicRootControlPlane.resolveLocalTopicRoot(topic, {
|
|
2174
|
+
signal: policySignal,
|
|
2175
|
+
});
|
|
2176
|
+
return {
|
|
2177
|
+
root: local ??
|
|
2178
|
+
this.topicRootControlPlane.resolveDeterministicTopicRoot(topic),
|
|
2179
|
+
autoCandidate: !local && this.autoTopicRootCandidates,
|
|
2180
|
+
};
|
|
1838
2181
|
});
|
|
2182
|
+
let resolved = await resolveCanonical();
|
|
2183
|
+
if (resolved.autoCandidate && topic.startsWith(this.shardTopicPrefix)) {
|
|
2184
|
+
if (await this.preflightRelayRootCandidates(topic, recovery, signal, isCurrent, true)) {
|
|
2185
|
+
throwIfAborted(signal);
|
|
2186
|
+
resolved = await resolveCanonical();
|
|
2187
|
+
}
|
|
2188
|
+
}
|
|
2189
|
+
const root = resolved.root;
|
|
1839
2190
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2191
|
+
if (!isCurrent())
|
|
2192
|
+
throw new AbortError("obsolete topic root query");
|
|
1840
2193
|
if (root !== this.publicKeyHash) {
|
|
1841
2194
|
return root;
|
|
1842
2195
|
}
|
|
@@ -1936,8 +2289,11 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1936
2289
|
async ensureFanoutChannel(shardTopic, options) {
|
|
1937
2290
|
const topic = shardTopic.toString();
|
|
1938
2291
|
const lifecycleRevision = this.topicControlPlaneLifecycleRevision;
|
|
2292
|
+
const recovery = {};
|
|
1939
2293
|
for (;;) {
|
|
1940
2294
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2295
|
+
if (options?.isCurrent?.() === false)
|
|
2296
|
+
throw new AbortError("obsolete fanout channel owner");
|
|
1941
2297
|
if (options?.signal?.aborted) {
|
|
1942
2298
|
throw (options.signal.reason ??
|
|
1943
2299
|
new AbortError("fanout channel opening aborted"));
|
|
@@ -1945,7 +2301,8 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1945
2301
|
const candidateGeneration = this.getTopicRootCandidateGeneration();
|
|
1946
2302
|
const active = this.ensureFanoutChannelInFlight.get(topic);
|
|
1947
2303
|
if (active?.candidateGeneration === candidateGeneration &&
|
|
1948
|
-
active.lifecycleRevision === lifecycleRevision
|
|
2304
|
+
active.lifecycleRevision === lifecycleRevision &&
|
|
2305
|
+
active.isCurrent()) {
|
|
1949
2306
|
try {
|
|
1950
2307
|
await withAbort(active.opening, options?.signal);
|
|
1951
2308
|
}
|
|
@@ -1953,6 +2310,10 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1953
2310
|
if (options?.signal?.aborted)
|
|
1954
2311
|
throw error;
|
|
1955
2312
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2313
|
+
if (candidateGeneration === this.getTopicRootCandidateGeneration() &&
|
|
2314
|
+
!active.abortController.signal.aborted &&
|
|
2315
|
+
active.isCurrent())
|
|
2316
|
+
throw error;
|
|
1956
2317
|
}
|
|
1957
2318
|
await withAbort(active.settled, options?.signal);
|
|
1958
2319
|
if (this.ensureFanoutChannelInFlight.get(topic) === active) {
|
|
@@ -1981,13 +2342,19 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1981
2342
|
once: true,
|
|
1982
2343
|
});
|
|
1983
2344
|
}
|
|
1984
|
-
|
|
2345
|
+
// Publish the opening identity before even a synchronously resolved root
|
|
2346
|
+
// can enter its guarded recovery/creation continuation.
|
|
2347
|
+
const opening = Promise.resolve().then(() => this.ensureFanoutChannelOnce(topic, { ...options, signal: abortController.signal }, lifecycleRevision, candidateGeneration, recovery, () => this.ensureFanoutChannelInFlight.get(topic)?.abortController ===
|
|
2348
|
+
abortController &&
|
|
2349
|
+
!abortController.signal.aborted &&
|
|
2350
|
+
options?.isCurrent?.() !== false));
|
|
1985
2351
|
let resolveSettled;
|
|
1986
2352
|
const settled = new Promise((resolve) => {
|
|
1987
2353
|
resolveSettled = resolve;
|
|
1988
2354
|
});
|
|
1989
2355
|
const inFlight = {
|
|
1990
2356
|
abortController,
|
|
2357
|
+
isCurrent: () => options?.isCurrent?.() !== false,
|
|
1991
2358
|
candidateGeneration,
|
|
1992
2359
|
lifecycleRevision,
|
|
1993
2360
|
opening,
|
|
@@ -2030,7 +2397,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
2030
2397
|
}
|
|
2031
2398
|
}
|
|
2032
2399
|
}
|
|
2033
|
-
async ensureFanoutChannelOnce(shardTopic, options, lifecycleRevision, candidateGeneration) {
|
|
2400
|
+
async ensureFanoutChannelOnce(shardTopic, options, lifecycleRevision, candidateGeneration, recovery = {}, isOpeningCurrent = () => true) {
|
|
2034
2401
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2035
2402
|
if (candidateGeneration !== this.getTopicRootCandidateGeneration())
|
|
2036
2403
|
return;
|
|
@@ -2049,15 +2416,16 @@ export class TopicControlPlane extends DirectStream {
|
|
|
2049
2416
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2050
2417
|
let resolvedGeneration = candidateGeneration;
|
|
2051
2418
|
let root = suppliedRoot;
|
|
2419
|
+
let autoCandidate = false;
|
|
2420
|
+
const resolveRoot = () => this.withRelayRootPolicyDeadline(recovery, options?.signal, (signal) => this.resolveShardRootState(t, { signal }));
|
|
2052
2421
|
const existing = this.fanoutChannels.get(t);
|
|
2053
2422
|
if (existing) {
|
|
2054
2423
|
if (!root) {
|
|
2055
|
-
const resolved = await
|
|
2056
|
-
signal: options?.signal,
|
|
2057
|
-
});
|
|
2424
|
+
const resolved = await resolveRoot();
|
|
2058
2425
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2059
2426
|
root = resolved.root;
|
|
2060
2427
|
resolvedGeneration = resolved.candidateGeneration;
|
|
2428
|
+
autoCandidate = resolved.autoCandidate;
|
|
2061
2429
|
}
|
|
2062
2430
|
if (resolvedGeneration !== candidateGeneration ||
|
|
2063
2431
|
candidateGeneration !== this.getTopicRootCandidateGeneration()) {
|
|
@@ -2091,15 +2459,30 @@ export class TopicControlPlane extends DirectStream {
|
|
|
2091
2459
|
}
|
|
2092
2460
|
}
|
|
2093
2461
|
if (!root) {
|
|
2094
|
-
const resolved = await
|
|
2095
|
-
signal: options?.signal,
|
|
2096
|
-
});
|
|
2462
|
+
const resolved = await resolveRoot();
|
|
2097
2463
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2098
2464
|
root = resolved.root;
|
|
2099
2465
|
resolvedGeneration = resolved.candidateGeneration;
|
|
2466
|
+
autoCandidate = resolved.autoCandidate;
|
|
2467
|
+
}
|
|
2468
|
+
if (autoCandidate &&
|
|
2469
|
+
(await this.preflightRelayRootCandidates(t, recovery, options?.signal, isOpeningCurrent))) {
|
|
2470
|
+
throwIfAborted(options?.signal);
|
|
2471
|
+
if (candidateGeneration !== this.getTopicRootCandidateGeneration())
|
|
2472
|
+
return;
|
|
2473
|
+
const resolved = await resolveRoot();
|
|
2474
|
+
root = resolved.root;
|
|
2475
|
+
resolvedGeneration = resolved.candidateGeneration;
|
|
2476
|
+
autoCandidate = resolved.autoCandidate;
|
|
2477
|
+
}
|
|
2478
|
+
if (suppliedRoot !== undefined ||
|
|
2479
|
+
!this.autoTopicRootCandidates ||
|
|
2480
|
+
autoCandidate) {
|
|
2481
|
+
root = await this.confirmDirectShardRoot(t, root, resolvedGeneration, lifecycleRevision, options?.signal);
|
|
2100
2482
|
}
|
|
2101
|
-
root = await this.confirmDirectShardRoot(t, root, resolvedGeneration, lifecycleRevision, options?.signal);
|
|
2102
2483
|
this.assertTopicControlPlaneActive(lifecycleRevision);
|
|
2484
|
+
if (!isOpeningCurrent())
|
|
2485
|
+
throw new AbortError("obsolete fanout channel owner");
|
|
2103
2486
|
if (resolvedGeneration !== candidateGeneration ||
|
|
2104
2487
|
candidateGeneration !== this.getTopicRootCandidateGeneration()) {
|
|
2105
2488
|
return;
|
|
@@ -2420,6 +2803,78 @@ export class TopicControlPlane extends DirectStream {
|
|
|
2420
2803
|
this.initializeTopic(topic);
|
|
2421
2804
|
return this.debounceSubscribeAggregator.add({ key: topic });
|
|
2422
2805
|
}
|
|
2806
|
+
async announceShardSubscriptions(shardTopic, lifecycleRevision) {
|
|
2807
|
+
const st = this.fanoutChannels.get(shardTopic);
|
|
2808
|
+
if (!st)
|
|
2809
|
+
return;
|
|
2810
|
+
let topics = [...this.subscriptions.entries()].filter(([topic]) => this.getShardTopicForUserTopic(topic) === shardTopic);
|
|
2811
|
+
while (topics.length > 0 &&
|
|
2812
|
+
this.isFanoutChannelCurrent(shardTopic, st.channel, lifecycleRevision)) {
|
|
2813
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(new Subscribe({
|
|
2814
|
+
topics: topics.map(([topic]) => topic),
|
|
2815
|
+
requestSubscribers: true,
|
|
2816
|
+
})), {
|
|
2817
|
+
mode: new AnyWhere(),
|
|
2818
|
+
priority: 1,
|
|
2819
|
+
skipRecipientValidation: true,
|
|
2820
|
+
});
|
|
2821
|
+
if (!this.isFanoutChannelCurrent(shardTopic, st.channel, lifecycleRevision))
|
|
2822
|
+
return;
|
|
2823
|
+
const current = topics.filter(([topic, sub]) => this.subscriptions.get(topic) === sub);
|
|
2824
|
+
if (current.length !== topics.length) {
|
|
2825
|
+
// Never announce a subscription removed while signing. This retry
|
|
2826
|
+
// only shrinks the original batch, so churn cannot extend it forever.
|
|
2827
|
+
topics = current;
|
|
2828
|
+
continue;
|
|
2829
|
+
}
|
|
2830
|
+
await st.channel.publish(toUint8Array(embedded.bytes()));
|
|
2831
|
+
if (this.isFanoutChannelCurrent(shardTopic, st.channel, lifecycleRevision)) {
|
|
2832
|
+
this.touchFanoutChannel(shardTopic, st.channel, lifecycleRevision);
|
|
2833
|
+
}
|
|
2834
|
+
return;
|
|
2835
|
+
}
|
|
2836
|
+
}
|
|
2837
|
+
async announceDirectSubscriptions(topics, peers = [...this.peers.values()], subscribed = true) {
|
|
2838
|
+
const revision = this.topicControlPlaneLifecycleRevision;
|
|
2839
|
+
const signal = this.topicRootResolutionAbortController.signal;
|
|
2840
|
+
const subscriptions = new Map(topics.map((topic) => [topic, this.subscriptions.get(topic)]));
|
|
2841
|
+
const isCurrentTopic = (topic) => subscribed
|
|
2842
|
+
? subscriptions.get(topic) != null &&
|
|
2843
|
+
this.subscriptions.get(topic) === subscriptions.get(topic)
|
|
2844
|
+
: !this.subscriptions.has(topic);
|
|
2845
|
+
await Promise.all(peers
|
|
2846
|
+
.map(async (peer) => {
|
|
2847
|
+
const isCurrent = () => this.isTopicControlPlaneActive(revision) &&
|
|
2848
|
+
this.peers.get(peer.publicKey.hashcode()) === peer &&
|
|
2849
|
+
peer.isWritable;
|
|
2850
|
+
let currentTopics = topics.filter(isCurrentTopic);
|
|
2851
|
+
while (isCurrent() && currentTopics.length > 0) {
|
|
2852
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(subscribed
|
|
2853
|
+
? new Subscribe({
|
|
2854
|
+
topics: currentTopics,
|
|
2855
|
+
requestSubscribers: true,
|
|
2856
|
+
})
|
|
2857
|
+
: new Unsubscribe({ topics: currentTopics })), {
|
|
2858
|
+
mode: new SilentDelivery({
|
|
2859
|
+
to: [peer.publicKey.hashcode()],
|
|
2860
|
+
redundancy: 1,
|
|
2861
|
+
}),
|
|
2862
|
+
priority: 1,
|
|
2863
|
+
skipRecipientValidation: true,
|
|
2864
|
+
});
|
|
2865
|
+
if (!isCurrent())
|
|
2866
|
+
return;
|
|
2867
|
+
const remaining = currentTopics.filter(isCurrentTopic);
|
|
2868
|
+
if (remaining.length !== currentTopics.length) {
|
|
2869
|
+
currentTopics = remaining;
|
|
2870
|
+
continue;
|
|
2871
|
+
}
|
|
2872
|
+
await this.publishMessage(this.publicKey, embedded, [peer], undefined, signal);
|
|
2873
|
+
return;
|
|
2874
|
+
}
|
|
2875
|
+
})
|
|
2876
|
+
.map((task) => task.catch(logErrorIfStarted)));
|
|
2877
|
+
}
|
|
2423
2878
|
async _subscribe(topics, lifecycleRevision = this.topicControlPlaneLifecycleRevision) {
|
|
2424
2879
|
if (!this.started)
|
|
2425
2880
|
throw new NotStartedError();
|
|
@@ -2428,7 +2883,6 @@ export class TopicControlPlane extends DirectStream {
|
|
|
2428
2883
|
if (topics.length === 0)
|
|
2429
2884
|
return;
|
|
2430
2885
|
const byShard = new Map();
|
|
2431
|
-
const joins = [];
|
|
2432
2886
|
for (const { key: topic, counter } of topics) {
|
|
2433
2887
|
let prev = this.subscriptions.get(topic);
|
|
2434
2888
|
if (prev) {
|
|
@@ -2442,33 +2896,25 @@ export class TopicControlPlane extends DirectStream {
|
|
|
2442
2896
|
const shardTopic = this.getShardTopicForUserTopic(topic);
|
|
2443
2897
|
byShard.set(shardTopic, [...(byShard.get(shardTopic) ?? []), topic]);
|
|
2444
2898
|
this.shardRefCounts.set(shardTopic, (this.shardRefCounts.get(shardTopic) ?? 0) + 1);
|
|
2445
|
-
joins.push(this.ensureFanoutChannel(shardTopic));
|
|
2446
2899
|
}
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
throw new Error(`Fanout channel missing for shard: ${shardTopic}`);
|
|
2457
|
-
const msg = new Subscribe({
|
|
2458
|
-
topics: userTopics,
|
|
2459
|
-
requestSubscribers: true,
|
|
2460
|
-
});
|
|
2461
|
-
const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
|
|
2462
|
-
mode: new AnyWhere(),
|
|
2463
|
-
priority: 1,
|
|
2464
|
-
skipRecipientValidation: true,
|
|
2900
|
+
// Neighbours can use different root policies. Discover them independently
|
|
2901
|
+
// of shard joins, without adopting or overriding either peer's root policy.
|
|
2902
|
+
void this.announceDirectSubscriptions([...byShard.values()].flat()).catch(logErrorIfStarted);
|
|
2903
|
+
const results = await Promise.allSettled([...byShard.keys()].map(async (shardTopic) => {
|
|
2904
|
+
const owners = byShard
|
|
2905
|
+
.get(shardTopic)
|
|
2906
|
+
.map((topic) => [topic, this.subscriptions.get(topic)]);
|
|
2907
|
+
await this.ensureFanoutChannel(shardTopic, {
|
|
2908
|
+
isCurrent: () => owners.some(([topic, owner]) => owner && this.subscriptions.get(topic) === owner),
|
|
2465
2909
|
});
|
|
2466
|
-
|
|
2467
|
-
return;
|
|
2468
|
-
await st.channel.publish(toUint8Array(embedded.bytes()));
|
|
2469
|
-
if (this.isFanoutChannelCurrent(shardTopic, st.channel, lifecycleRevision))
|
|
2470
|
-
this.touchFanoutChannel(shardTopic, st.channel, lifecycleRevision);
|
|
2910
|
+
await this.announceShardSubscriptions(shardTopic, lifecycleRevision);
|
|
2471
2911
|
}));
|
|
2912
|
+
const failure = results.find((result) => result.status === "rejected");
|
|
2913
|
+
if (failure?.status === "rejected") {
|
|
2914
|
+
if (this.isTopicControlPlaneActive(lifecycleRevision))
|
|
2915
|
+
this.scheduleReconcileShardOverlays();
|
|
2916
|
+
throw failure.reason;
|
|
2917
|
+
}
|
|
2472
2918
|
}
|
|
2473
2919
|
async unsubscribe(topic, options) {
|
|
2474
2920
|
this.pendingSubscriptions.delete(topic);
|
|
@@ -2500,6 +2946,10 @@ export class TopicControlPlane extends DirectStream {
|
|
|
2500
2946
|
const next = (this.shardRefCounts.get(shardTopic) ?? 0) - 1;
|
|
2501
2947
|
if (next <= 0) {
|
|
2502
2948
|
this.shardRefCounts.delete(shardTopic);
|
|
2949
|
+
const opening = this.ensureFanoutChannelInFlight.get(shardTopic);
|
|
2950
|
+
if (opening && !opening.isCurrent()) {
|
|
2951
|
+
opening.abortController.abort(new AbortError("last shard subscription removed"));
|
|
2952
|
+
}
|
|
2503
2953
|
}
|
|
2504
2954
|
else {
|
|
2505
2955
|
this.shardRefCounts.set(shardTopic, next);
|
|
@@ -2517,12 +2967,14 @@ export class TopicControlPlane extends DirectStream {
|
|
|
2517
2967
|
return;
|
|
2518
2968
|
const byShard = new Map();
|
|
2519
2969
|
for (const { key: topic } of topics) {
|
|
2520
|
-
//
|
|
2970
|
+
// A pending reopen must not erase the previous departure: its Subscribe
|
|
2971
|
+
// shares the transport session and would otherwise only refresh the cache.
|
|
2521
2972
|
if (this.subscriptions.has(topic))
|
|
2522
2973
|
continue;
|
|
2523
2974
|
const shardTopic = this.getShardTopicForUserTopic(topic);
|
|
2524
2975
|
byShard.set(shardTopic, [...(byShard.get(shardTopic) ?? []), topic]);
|
|
2525
2976
|
}
|
|
2977
|
+
void this.announceDirectSubscriptions([...byShard.values()].flat(), undefined, false).catch(logErrorIfStarted);
|
|
2526
2978
|
await Promise.all([...byShard.entries()].map(async ([shardTopic, userTopics]) => {
|
|
2527
2979
|
if (userTopics.length === 0)
|
|
2528
2980
|
return;
|
|
@@ -2881,6 +3333,16 @@ export class TopicControlPlane extends DirectStream {
|
|
|
2881
3333
|
}
|
|
2882
3334
|
onPeerUnreachable(publicKeyHash) {
|
|
2883
3335
|
super.onPeerUnreachable(publicKeyHash);
|
|
3336
|
+
const claim = this.signedTopicRootCandidateClaims.get(publicKeyHash);
|
|
3337
|
+
if (this.isTopicControlPlaneActive(this.topicControlPlaneLifecycleRevision) &&
|
|
3338
|
+
!this.maybeDisableAutoTopicRootCandidatesIfExternallyConfigured() &&
|
|
3339
|
+
claim &&
|
|
3340
|
+
!this.routes.isReachable(this.publicKeyHash, publicKeyHash) &&
|
|
3341
|
+
this.suppressTopicRootCandidateClaim(publicKeyHash, claim)) {
|
|
3342
|
+
this.rebuildAutoTopicRootCandidatesFromClaims(BigInt(Date.now()), {
|
|
3343
|
+
immediate: true,
|
|
3344
|
+
});
|
|
3345
|
+
}
|
|
2884
3346
|
const key = this.peerKeyHashToPublicKey.get(publicKeyHash);
|
|
2885
3347
|
if (!key) {
|
|
2886
3348
|
return;
|
|
@@ -3164,14 +3626,20 @@ export class TopicControlPlane extends DirectStream {
|
|
|
3164
3626
|
}
|
|
3165
3627
|
async processDirectPubSubMessage(input) {
|
|
3166
3628
|
const { pubsubMessage, message, from, stream } = input;
|
|
3167
|
-
|
|
3629
|
+
if ((pubsubMessage instanceof Subscribe ||
|
|
3630
|
+
pubsubMessage instanceof Unsubscribe) &&
|
|
3631
|
+
this.peers.get(stream.publicKey.hashcode()) !== stream)
|
|
3632
|
+
return;
|
|
3633
|
+
const isBoundDirectControl = pubsubMessage instanceof TopicRootCandidates ||
|
|
3168
3634
|
pubsubMessage instanceof TopicRootCandidateClaims ||
|
|
3169
3635
|
pubsubMessage instanceof TopicRootQuery ||
|
|
3170
|
-
pubsubMessage instanceof TopicRootQueryResponse
|
|
3171
|
-
|
|
3636
|
+
pubsubMessage instanceof TopicRootQueryResponse ||
|
|
3637
|
+
pubsubMessage instanceof Subscribe ||
|
|
3638
|
+
pubsubMessage instanceof Unsubscribe;
|
|
3639
|
+
const directRootSigner = isBoundDirectControl
|
|
3172
3640
|
? message.header.signatures?.publicKeys[0]
|
|
3173
3641
|
: undefined;
|
|
3174
|
-
if (
|
|
3642
|
+
if (isBoundDirectControl &&
|
|
3175
3643
|
(!directRootSigner || !directRootSigner.equals(stream.publicKey))) {
|
|
3176
3644
|
return;
|
|
3177
3645
|
}
|
|
@@ -3189,14 +3657,21 @@ export class TopicControlPlane extends DirectStream {
|
|
|
3189
3657
|
return;
|
|
3190
3658
|
}
|
|
3191
3659
|
const imported = [];
|
|
3660
|
+
const batchUpdate = { immediate: false };
|
|
3661
|
+
const revision = this.topicControlPlaneLifecycleRevision;
|
|
3192
3662
|
const outerPeerHash = stream.publicKey.hashcode();
|
|
3193
3663
|
for (const claim of pubsubMessage.claims) {
|
|
3194
|
-
if (await this.importSignedTopicRootCandidateClaim(claim, outerPeerHash, stream)) {
|
|
3664
|
+
if (await this.importSignedTopicRootCandidateClaim(claim, outerPeerHash, stream, batchUpdate)) {
|
|
3195
3665
|
imported.push(claim);
|
|
3196
3666
|
}
|
|
3197
3667
|
}
|
|
3668
|
+
if (!this.isTopicControlPlaneActive(revision) ||
|
|
3669
|
+
this.peers.get(outerPeerHash) !== stream)
|
|
3670
|
+
return;
|
|
3671
|
+
if (imported.length > 0 || batchUpdate.immediate) {
|
|
3672
|
+
this.rebuildAutoTopicRootCandidatesFromClaims(BigInt(Date.now()), batchUpdate);
|
|
3673
|
+
}
|
|
3198
3674
|
if (imported.length > 0) {
|
|
3199
|
-
this.rebuildAutoTopicRootCandidatesFromClaims();
|
|
3200
3675
|
// Relay the exact nested claims; no relay signature is substituted for
|
|
3201
3676
|
// their origin signatures.
|
|
3202
3677
|
void this.sendSignedTopicRootCandidateClaims(imported, [...this.peers.values()].filter((peer) => peer.publicKey.hashcode() !== outerPeerHash)).catch(() => { });
|
|
@@ -3204,7 +3679,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
3204
3679
|
return;
|
|
3205
3680
|
}
|
|
3206
3681
|
if (pubsubMessage instanceof TopicRootQuery) {
|
|
3207
|
-
const root = await this.resolveQueryableTopicRoot(pubsubMessage.topic);
|
|
3682
|
+
const root = await this.resolveQueryableTopicRoot(pubsubMessage.topic, () => this.peers.get(from.hashcode()) === stream);
|
|
3208
3683
|
await this.sendDirectControlMessage(stream, new TopicRootQueryResponse({
|
|
3209
3684
|
requestId: pubsubMessage.requestId,
|
|
3210
3685
|
topic: pubsubMessage.topic,
|
|
@@ -3265,6 +3740,10 @@ export class TopicControlPlane extends DirectStream {
|
|
|
3265
3740
|
}
|
|
3266
3741
|
return;
|
|
3267
3742
|
}
|
|
3743
|
+
if (pubsubMessage instanceof Unsubscribe) {
|
|
3744
|
+
this.processUnsubscribeMessage(message, pubsubMessage, from);
|
|
3745
|
+
return;
|
|
3746
|
+
}
|
|
3268
3747
|
if (pubsubMessage instanceof GetSubscribers) {
|
|
3269
3748
|
const overlap = this.getSubscriptionOverlap(pubsubMessage.topics);
|
|
3270
3749
|
if (overlap.length === 0)
|
|
@@ -3285,6 +3764,35 @@ export class TopicControlPlane extends DirectStream {
|
|
|
3285
3764
|
return;
|
|
3286
3765
|
}
|
|
3287
3766
|
}
|
|
3767
|
+
processUnsubscribeMessage(message, unsubscribe, from) {
|
|
3768
|
+
const senderKey = from.hashcode();
|
|
3769
|
+
const relevantTopics = unsubscribe.topics.filter((topic) => this.isTrackedTopic(topic));
|
|
3770
|
+
if (!relevantTopics.length ||
|
|
3771
|
+
!this.subscriptionMessageIsLatest(message, unsubscribe, relevantTopics))
|
|
3772
|
+
return;
|
|
3773
|
+
const changed = [];
|
|
3774
|
+
for (const topic of relevantTopics) {
|
|
3775
|
+
if (this.topics.get(topic)?.delete(senderKey)) {
|
|
3776
|
+
changed.push(topic);
|
|
3777
|
+
this.peerToTopic.get(senderKey)?.delete(topic);
|
|
3778
|
+
}
|
|
3779
|
+
}
|
|
3780
|
+
if (!this.peerToTopic.get(senderKey)?.size) {
|
|
3781
|
+
this.peerToTopic.delete(senderKey);
|
|
3782
|
+
}
|
|
3783
|
+
// Independent direct/shard paths may deliver an older Subscribe later.
|
|
3784
|
+
// Retain recent unsubscribe floors, bounded separately from live subscribers.
|
|
3785
|
+
const messages = this.lastSubscriptionMessages.get(senderKey);
|
|
3786
|
+
this.lastSubscriptionMessages.delete(senderKey);
|
|
3787
|
+
this.lastSubscriptionMessages.set(senderKey, messages);
|
|
3788
|
+
for (const topic of relevantTopics)
|
|
3789
|
+
this.pruneUnsubscribedWatermarks(topic);
|
|
3790
|
+
if (changed.length > 0) {
|
|
3791
|
+
this.dispatchEvent(new CustomEvent("unsubscribe", {
|
|
3792
|
+
detail: new UnsubcriptionEvent(from, changed, "remote-unsubscribe"),
|
|
3793
|
+
}));
|
|
3794
|
+
}
|
|
3795
|
+
}
|
|
3288
3796
|
async processShardPubSubMessage(input) {
|
|
3289
3797
|
const { pubsubMessage, message, from, shardTopic, isCurrent } = input;
|
|
3290
3798
|
if (isCurrent?.() === false)
|
|
@@ -3346,31 +3854,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
3346
3854
|
return;
|
|
3347
3855
|
}
|
|
3348
3856
|
if (pubsubMessage instanceof Unsubscribe) {
|
|
3349
|
-
|
|
3350
|
-
const senderKey = sender.hashcode();
|
|
3351
|
-
const relevantTopics = pubsubMessage.topics.filter((t) => this.isTrackedTopic(t));
|
|
3352
|
-
if (relevantTopics.length > 0 &&
|
|
3353
|
-
this.subscriptionMessageIsLatest(message, pubsubMessage, relevantTopics)) {
|
|
3354
|
-
const changed = [];
|
|
3355
|
-
for (const topic of relevantTopics) {
|
|
3356
|
-
const peers = this.topics.get(topic);
|
|
3357
|
-
if (!peers)
|
|
3358
|
-
continue;
|
|
3359
|
-
if (peers.delete(senderKey)) {
|
|
3360
|
-
changed.push(topic);
|
|
3361
|
-
this.peerToTopic.get(senderKey)?.delete(topic);
|
|
3362
|
-
}
|
|
3363
|
-
}
|
|
3364
|
-
if (!this.peerToTopic.get(senderKey)?.size) {
|
|
3365
|
-
this.peerToTopic.delete(senderKey);
|
|
3366
|
-
this.lastSubscriptionMessages.delete(senderKey);
|
|
3367
|
-
}
|
|
3368
|
-
if (changed.length > 0) {
|
|
3369
|
-
this.dispatchEvent(new CustomEvent("unsubscribe", {
|
|
3370
|
-
detail: new UnsubcriptionEvent(sender, changed, "remote-unsubscribe"),
|
|
3371
|
-
}));
|
|
3372
|
-
}
|
|
3373
|
-
}
|
|
3857
|
+
this.processUnsubscribeMessage(message, pubsubMessage, from);
|
|
3374
3858
|
return;
|
|
3375
3859
|
}
|
|
3376
3860
|
if (pubsubMessage instanceof PeerUnavailable) {
|
|
@@ -3463,6 +3947,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
3463
3947
|
!(pubsubMessage instanceof TopicRootQuery) &&
|
|
3464
3948
|
!(pubsubMessage instanceof GetSubscribers) &&
|
|
3465
3949
|
!(pubsubMessage instanceof Subscribe) &&
|
|
3950
|
+
!(pubsubMessage instanceof Unsubscribe) &&
|
|
3466
3951
|
!(pubsubMessage instanceof TopicRootQueryResponse)) {
|
|
3467
3952
|
return true;
|
|
3468
3953
|
}
|