@peerbit/pubsub 5.0.8 → 5.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/fanout-channel.d.ts +1 -0
- package/dist/src/fanout-channel.d.ts.map +1 -1
- package/dist/src/fanout-channel.js +3 -0
- package/dist/src/fanout-channel.js.map +1 -1
- package/dist/src/fanout-tree.d.ts +2 -0
- package/dist/src/fanout-tree.d.ts.map +1 -1
- package/dist/src/fanout-tree.js +24 -3
- package/dist/src/fanout-tree.js.map +1 -1
- package/dist/src/index.d.ts +10 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +232 -25
- package/dist/src/index.js.map +1 -1
- package/dist/src/topic-root-control-plane.d.ts +4 -0
- package/dist/src/topic-root-control-plane.d.ts.map +1 -1
- package/dist/src/topic-root-control-plane.js +17 -3
- package/dist/src/topic-root-control-plane.js.map +1 -1
- package/package.json +4 -4
- package/src/fanout-channel.ts +4 -0
- package/src/fanout-tree.ts +30 -4
- package/src/index.ts +307 -26
- package/src/topic-root-control-plane.ts +24 -3
package/dist/src/index.d.ts
CHANGED
|
@@ -134,6 +134,8 @@ export declare class TopicControlPlane extends DirectStream<PubSubEvents> implem
|
|
|
134
134
|
private autoCandidatesGossipUntil;
|
|
135
135
|
private _onFanoutPeerUnreachable?;
|
|
136
136
|
private fanoutChannels;
|
|
137
|
+
private pendingTopicRootQueries;
|
|
138
|
+
private nextTopicRootQueryId;
|
|
137
139
|
constructor(components: TopicControlPlaneComponents, props?: TopicControlPlaneOptions);
|
|
138
140
|
/**
|
|
139
141
|
* Configure deterministic topic-root candidates and disable the pubsub "auto"
|
|
@@ -171,7 +173,15 @@ export declare class TopicControlPlane extends DirectStream<PubSubEvents> implem
|
|
|
171
173
|
private touchFanoutChannel;
|
|
172
174
|
private evictEphemeralFanoutChannels;
|
|
173
175
|
private getShardTopicForUserTopic;
|
|
176
|
+
private resolveTopicRootState;
|
|
177
|
+
resolveTopicRoot(topic: string): Promise<string | undefined>;
|
|
174
178
|
private resolveShardRoot;
|
|
179
|
+
private getConnectedTopicRootTrackers;
|
|
180
|
+
private nextTopicRootRequestIdValue;
|
|
181
|
+
private sendDirectControlMessage;
|
|
182
|
+
private resolvePendingTopicRootQuery;
|
|
183
|
+
private queryTopicRootFromPeer;
|
|
184
|
+
private resolveTopicRootThroughPeers;
|
|
175
185
|
private ensureFanoutChannel;
|
|
176
186
|
private closeFanoutChannel;
|
|
177
187
|
hostShardRootsNow(): Promise<void>;
|
package/dist/src/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,KAAK,UAAU,EACf,KAAK,MAAM,IAAI,YAAY,EAC3B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAA0B,MAAM,iBAAiB,CAAC;AAExE,OAAO,EAIN,KAAK,MAAM,EAEX,KAAK,YAAY,EAIjB,gBAAgB,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,KAAK,UAAU,EACf,KAAK,MAAM,IAAI,YAAY,EAC3B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAA0B,MAAM,iBAAiB,CAAC;AAExE,OAAO,EAIN,KAAK,MAAM,EAEX,KAAK,YAAY,EAIjB,gBAAgB,EAQhB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACN,YAAY,EACZ,KAAK,sBAAsB,EAC3B,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAEN,mBAAmB,EAEnB,WAAW,EAEX,KAAK,gBAAgB,EACrB,KAAK,SAAS,EAGd,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAC5B,KAAK,SAAS,EACd,cAAc,EACd,KAAK,gBAAgB,EAGrB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAMhD,OAAO,KAAK,EACX,UAAU,EACV,wBAAwB,EAExB,qBAAqB,EACrB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAEtE,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC;AACpC,cAAc,+BAA+B,CAAC;AAE9C,eAAO,MAAM,YAAY,GAAI,KAAK,cAAc,GAAG,UAAU,gCACR,CAAC;AAEtD,eAAO,MAAM,MAAM,oCAAoD,CAAC;AA+DxE,MAAM,MAAM,wBAAwB,GAAG,mBAAmB,GAAG;IAC5D,0BAA0B,CAAC,EAAE,OAAO,CAAC;IACrC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C;;OAEG;IACH,MAAM,EAAE,UAAU,CAAC;IACnB;;OAEG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE,MAAM,CAAC,CAAC,CAAC;IAChE;;OAEG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE,MAAM,CAAC,CAAC,CAAC;IACpE;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE,MAAM,CAAC,CAAC,CAAC;IACpE;;OAEG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;OAIG;IACH,8BAA8B,CAAC,EAAE,OAAO,CAAC;IACzC;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC;;;;OAIG;IACH,iCAAiC,CAAC,EAAE,MAAM,CAAC;IAC3C;;;;;OAKG;IACH,yBAAyB,CAAC,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,sBAAsB,CAAC;AAEjE,MAAM,MAAM,MAAM,GAAG,YAAY,GAAG,aAAa,CAAC;AAWlD;;GAEG;AACH,qBAAa,iBACZ,SAAQ,YAAY,CAAC,YAAY,CACjC,YAAW,MAAM;IAGV,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAEnD,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAEtC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAEvD,OAAO,CAAC,oBAAoB,CAAc;IACnC,wBAAwB,EAAE,GAAG,CACnC,MAAM,EACN,GAAG,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CACnD,CAAa;IACP,0BAA0B,EAAE,OAAO,CAAC;IAC3C,SAAgB,qBAAqB,EAAE,qBAAqB,CAAC;IAC7D,SAAgB,yBAAyB,EAAE,MAAM,CAAC;IAClD,SAAgB,MAAM,EAAE,UAAU,CAAC;IAEnC,OAAO,CAAC,2BAA2B,CAAiC;IACpE,OAAO,CAAC,6BAA6B,CAAiC;IAEtE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;IACrC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAG3B;IACJ,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA6B;IAC7D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA6B;IAC5D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAqB;IAElD,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAGvC;IACF,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAGvC;IACF,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,8BAA8B,CAAU;IACzD,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAS;IAK3D,OAAO,CAAC,uBAAuB,CAAS;IACxC,OAAO,CAAC,yBAAyB,CAAC,CAAc;IAChD,OAAO,CAAC,8BAA8B,CAAC,CAAgB;IACvD,OAAO,CAAC,2BAA2B,CAAC,CAAgB;IACpD,OAAO,CAAC,wBAAwB,CAAS;IACzC,OAAO,CAAC,6BAA6B,CACjC;IACJ,OAAO,CAAC,4BAA4B,CAAC,CAAiC;IACtE,OAAO,CAAC,yBAAyB,CAAK;IACtC,OAAO,CAAC,wBAAwB,CAAC,CAEvB;IAEV,OAAO,CAAC,cAAc,CAYlB;IACJ,OAAO,CAAC,uBAAuB,CAO3B;IACJ,OAAO,CAAC,oBAAoB,CAAK;gBAGhC,UAAU,EAAE,2BAA2B,EACvC,KAAK,CAAC,EAAE,wBAAwB;IAuFjC;;;;;;;;;OASG;IACI,sBAAsB,CAAC,UAAU,EAAE,MAAM,EAAE;IAwB5B,KAAK;IAmBL,IAAI;IA4DJ,eAAe,CACpC,MAAM,EAAE,YAAY,EACpB,UAAU,EAAE,UAAU;IAoBP,OAAO,CACtB,MAAM,EAAE,YAAY,EACpB,SAAS,EAAE,aAAa,EACxB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,GACZ,WAAW;IASd,OAAO,CAAC,yDAAyD;IA0BjE,OAAO,CAAC,kCAAkC;IAiC1C,OAAO,CAAC,gCAAgC;IAWxC,OAAO,CAAC,wCAAwC;IA4BhD,OAAO,CAAC,kCAAkC;YAsB5B,2BAA2B;IAiBzC,OAAO,CAAC,oCAAoC;IA6B5C,OAAO,CAAC,2BAA2B;IAyBnC,OAAO,CAAC,8BAA8B;YAcxB,sBAAsB;IAiCpC,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,YAAY;IAcpB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,qBAAqB;IAiB7B,OAAO,CAAC,sBAAsB;IAyB9B,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,uBAAuB;IAiB/B,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,4BAA4B;IAwBpC,OAAO,CAAC,yBAAyB;YAUnB,qBAAqB;IA+BtB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;YAI3D,gBAAgB;IA8B9B,OAAO,CAAC,6BAA6B;IAYrC,OAAO,CAAC,2BAA2B;YAYrB,wBAAwB;IAetC,OAAO,CAAC,4BAA4B;YAatB,sBAAsB;YAiCtB,4BAA4B;YAuB5B,mBAAmB;YA2MnB,kBAAkB;IAgCnB,iBAAiB;IAYxB,SAAS,CAAC,KAAK,EAAE,MAAM;YAQf,UAAU;IAkDlB,WAAW,CAChB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QACT,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,IAAI,CAAC,EAAE,UAAU,CAAC;KAClB;YA0CY,oBAAoB;YAgDpB,uBAAuB;YA6DvB,8BAA8B;IA4B5C,OAAO,CAAC,uBAAuB;IAQ/B,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,EAAE,GAAG,SAAS;IAa1D;;;;OAIG;IACH,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,SAAS,EAAE;IA4B9D,kBAAkB,CACvB,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,EACxB,EAAE,CAAC,EAAE,aAAa,GAChB,OAAO,CAAC,IAAI,CAAC;IAuDV,OAAO,CACZ,IAAI,EAAE,UAAU,GAAG,SAAS,EAC5B,OAAO,CAAC,EAAE;QACT,MAAM,EAAE,MAAM,EAAE,CAAC;KACjB,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG;QACzB,IAAI,CAAC,EAAE,cAAc,GAAG,mBAAmB,CAAC;KAC5C,GAAG,eAAe,GAClB,uBAAuB,GACvB,gBAAgB,GAChB,SAAS,GACT,gBAAgB,GAAG;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAC1C,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IA0H3B,aAAa,CAAC,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAWhD,iBAAiB,CAAC,aAAa,EAAE,MAAM;IA2BvD,OAAO,CAAC,wBAAwB;IA2BhC,OAAO,CAAC,gCAAgC;IAoCxC,OAAO,CAAC,mBAAmB;IA8B3B,OAAO,CAAC,yBAAyB;IAuCjC,OAAO,CAAC,2BAA2B;YAcrB,4BAA4B;YA+B5B,0BAA0B;YA4H1B,yBAAyB;IA6NjB,aAAa,CAClC,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,MAAM;CAqFnB;AAED,eAAO,MAAM,kBAAkB,GAC9B,QAAQ;IAAE,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,EACxC,aACG,MAAM,GACN,MAAM,EAAE,GACR;IAAE,MAAM,EAAE,YAAY,CAAA;CAAE,GACxB;IAAE,MAAM,EAAE,YAAY,CAAA;CAAE,EAAE,GAC1B,MAAM,GACN,MAAM,EAAE,EACX,OAAO,MAAM,EACb,UAAU;IAAE,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,kBAiKpD,CAAC"}
|
package/dist/src/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import {} from "@libp2p/interface";
|
|
2
2
|
import { PublicSignKey, getPublicKeyFromPeerId } from "@peerbit/crypto";
|
|
3
3
|
import { logger as loggerFn } from "@peerbit/logger";
|
|
4
|
-
import { DataEvent, GetSubscribers, PeerUnavailable, PubSubData, PubSubMessage, PublishEvent, Subscribe, SubscriptionData, SubscriptionEvent, TopicRootCandidates, UnsubcriptionEvent, Unsubscribe, } from "@peerbit/pubsub-interface";
|
|
5
|
-
import { DirectStream,
|
|
4
|
+
import { DataEvent, GetSubscribers, PeerUnavailable, PubSubData, PubSubMessage, PublishEvent, Subscribe, SubscriptionData, SubscriptionEvent, TopicRootCandidates, TopicRootQuery, TopicRootQueryResponse, UnsubcriptionEvent, Unsubscribe, } from "@peerbit/pubsub-interface";
|
|
5
|
+
import { DirectStream, } from "@peerbit/stream";
|
|
6
6
|
import { AcknowledgeAnyWhere, AcknowledgeDelivery, AnyWhere, DataMessage, DeliveryError, MessageHeader, NotStartedError, SilentDelivery, deliveryModeHasReceiver, getMsgId, } from "@peerbit/stream-interface";
|
|
7
7
|
import { AbortError, TimeoutError, delay } from "@peerbit/time";
|
|
8
8
|
import { Uint8ArrayList } from "uint8arraylist";
|
|
@@ -58,6 +58,9 @@ const DEFAULT_PUBSUB_SHARD_COUNT = 256;
|
|
|
58
58
|
const PUBSUB_SHARD_COUNT_HARD_CAP = 16_384;
|
|
59
59
|
const DEFAULT_PUBSUB_SHARD_TOPIC_PREFIX = "/peerbit/pubsub-shard/1/";
|
|
60
60
|
const AUTO_TOPIC_ROOT_CANDIDATES_MAX = 64;
|
|
61
|
+
// Topic-root queries may need to wait for the responder to finish opening an
|
|
62
|
+
// outbound stream back to the requester after an inbound-only dial.
|
|
63
|
+
const DEFAULT_TOPIC_ROOT_QUERY_TIMEOUT_MS = 12_000;
|
|
61
64
|
const DEFAULT_PUBSUB_FANOUT_CHANNEL_OPTIONS = {
|
|
62
65
|
msgRate: 30,
|
|
63
66
|
msgSize: 1024,
|
|
@@ -118,6 +121,8 @@ export class TopicControlPlane extends DirectStream {
|
|
|
118
121
|
autoCandidatesGossipUntil = 0;
|
|
119
122
|
_onFanoutPeerUnreachable;
|
|
120
123
|
fanoutChannels = new Map();
|
|
124
|
+
pendingTopicRootQueries = new Map();
|
|
125
|
+
nextTopicRootQueryId = 1;
|
|
121
126
|
constructor(components, props) {
|
|
122
127
|
super(components, ["/peerbit/topic-control-plane/2.0.0"], props);
|
|
123
128
|
this.subscriptions = new Map();
|
|
@@ -269,6 +274,11 @@ export class TopicControlPlane extends DirectStream {
|
|
|
269
274
|
this.shardTopicCache.clear();
|
|
270
275
|
this.shardRefCounts.clear();
|
|
271
276
|
this.pinnedShards.clear();
|
|
277
|
+
for (const pending of this.pendingTopicRootQueries.values()) {
|
|
278
|
+
clearTimeout(pending.timer);
|
|
279
|
+
pending.resolve(undefined);
|
|
280
|
+
}
|
|
281
|
+
this.pendingTopicRootQueries.clear();
|
|
272
282
|
this.debounceSubscribeAggregator.close();
|
|
273
283
|
this.debounceUnsubscribeAggregator.close();
|
|
274
284
|
return super.stop();
|
|
@@ -424,7 +434,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
424
434
|
priority: 1,
|
|
425
435
|
skipRecipientValidation: true,
|
|
426
436
|
});
|
|
427
|
-
await this.
|
|
437
|
+
await this.publishMessageMaybe(this.publicKey, embedded, streams);
|
|
428
438
|
}
|
|
429
439
|
mergeAutoTopicRootCandidatesFromPeer(candidates) {
|
|
430
440
|
if (!this.autoTopicRootCandidates)
|
|
@@ -665,6 +675,32 @@ export class TopicControlPlane extends DirectStream {
|
|
|
665
675
|
this.shardTopicCache.set(t, shardTopic);
|
|
666
676
|
return shardTopic;
|
|
667
677
|
}
|
|
678
|
+
async resolveTopicRootState(topic) {
|
|
679
|
+
const tracked = await this.topicRootControlPlane.resolveTrackedTopicRoot(topic);
|
|
680
|
+
if (tracked) {
|
|
681
|
+
return { root: tracked, authoritative: true };
|
|
682
|
+
}
|
|
683
|
+
const resolvedThroughPeers = await this.resolveTopicRootThroughPeers(topic);
|
|
684
|
+
if (resolvedThroughPeers) {
|
|
685
|
+
return { root: resolvedThroughPeers, authoritative: true };
|
|
686
|
+
}
|
|
687
|
+
const deterministic = this.topicRootControlPlane.resolveDeterministicTopicRoot(topic);
|
|
688
|
+
if (deterministic === this.publicKeyHash &&
|
|
689
|
+
this.autoTopicRootCandidates &&
|
|
690
|
+
this.getConnectedTopicRootTrackers().length > 0) {
|
|
691
|
+
for (let attempt = 0; attempt < 8; attempt++) {
|
|
692
|
+
await delay(150 * (attempt < 4 ? 1 : 2));
|
|
693
|
+
const retried = await this.resolveTopicRootThroughPeers(topic);
|
|
694
|
+
if (retried) {
|
|
695
|
+
return { root: retried, authoritative: true };
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
return { root: deterministic, authoritative: false };
|
|
700
|
+
}
|
|
701
|
+
async resolveTopicRoot(topic) {
|
|
702
|
+
return (await this.resolveTopicRootState(topic)).root;
|
|
703
|
+
}
|
|
668
704
|
async resolveShardRoot(shardTopic) {
|
|
669
705
|
// If someone configured topic-root candidates externally (e.g. TestSession router
|
|
670
706
|
// selection or Peerbit.bootstrap) after this peer entered auto mode, disable auto
|
|
@@ -672,15 +708,106 @@ export class TopicControlPlane extends DirectStream {
|
|
|
672
708
|
if (this.autoTopicRootCandidates) {
|
|
673
709
|
this.maybeDisableAutoTopicRootCandidatesIfExternallyConfigured();
|
|
674
710
|
}
|
|
711
|
+
const hasConnectedTrackers = this.getConnectedTopicRootTrackers().length > 0;
|
|
675
712
|
const cached = this.shardRootCache.get(shardTopic);
|
|
676
|
-
if (cached)
|
|
677
|
-
return cached;
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
713
|
+
if (cached && (cached.authoritative || !hasConnectedTrackers)) {
|
|
714
|
+
return cached.root;
|
|
715
|
+
}
|
|
716
|
+
const resolved = await this.resolveTopicRootState(shardTopic);
|
|
717
|
+
if (!resolved.root) {
|
|
718
|
+
throw new Error(`No root resolved for shard topic ${shardTopic}. Configure TopicRootControlPlane candidates/resolver/trackers, or dial/bootstrap a peer that can resolve shard roots.`);
|
|
719
|
+
}
|
|
720
|
+
if (resolved.authoritative || !hasConnectedTrackers) {
|
|
721
|
+
this.shardRootCache.set(shardTopic, resolved);
|
|
722
|
+
}
|
|
723
|
+
else {
|
|
724
|
+
this.shardRootCache.delete(shardTopic);
|
|
725
|
+
}
|
|
726
|
+
return resolved.root;
|
|
727
|
+
}
|
|
728
|
+
getConnectedTopicRootTrackers() {
|
|
729
|
+
return [...this.peers.values()]
|
|
730
|
+
.filter((peer) => peer.isReadable || peer.isWritable)
|
|
731
|
+
.sort((a, b) => a.publicKey.hashcode() < b.publicKey.hashcode()
|
|
732
|
+
? -1
|
|
733
|
+
: a.publicKey.hashcode() > b.publicKey.hashcode()
|
|
734
|
+
? 1
|
|
735
|
+
: 0);
|
|
736
|
+
}
|
|
737
|
+
nextTopicRootRequestIdValue() {
|
|
738
|
+
let requestId = this.nextTopicRootQueryId >>> 0;
|
|
739
|
+
do {
|
|
740
|
+
requestId = requestId === 0 ? 1 : requestId;
|
|
741
|
+
this.nextTopicRootQueryId = ((requestId + 1) >>> 0) || 1;
|
|
742
|
+
if (!this.pendingTopicRootQueries.has(requestId)) {
|
|
743
|
+
return requestId;
|
|
744
|
+
}
|
|
745
|
+
requestId = this.nextTopicRootQueryId >>> 0;
|
|
746
|
+
} while (true);
|
|
747
|
+
}
|
|
748
|
+
async sendDirectControlMessage(peer, pubsubMessage) {
|
|
749
|
+
const embedded = await this.createMessage(toUint8Array(pubsubMessage.bytes()), {
|
|
750
|
+
mode: new SilentDelivery({
|
|
751
|
+
to: [peer.publicKey.hashcode()],
|
|
752
|
+
redundancy: 1,
|
|
753
|
+
}),
|
|
754
|
+
priority: 1,
|
|
755
|
+
skipRecipientValidation: true,
|
|
756
|
+
});
|
|
757
|
+
await this.publishMessage(this.publicKey, embedded, [peer]);
|
|
758
|
+
}
|
|
759
|
+
resolvePendingTopicRootQuery(message) {
|
|
760
|
+
const pending = this.pendingTopicRootQueries.get(message.requestId);
|
|
761
|
+
if (!pending || pending.topic !== message.topic) {
|
|
762
|
+
return false;
|
|
763
|
+
}
|
|
764
|
+
this.pendingTopicRootQueries.delete(message.requestId);
|
|
765
|
+
clearTimeout(pending.timer);
|
|
766
|
+
pending.resolve(message.root);
|
|
767
|
+
return true;
|
|
768
|
+
}
|
|
769
|
+
async queryTopicRootFromPeer(peer, topic) {
|
|
770
|
+
if (!this.started || this.stopping)
|
|
771
|
+
return undefined;
|
|
772
|
+
const requestId = this.nextTopicRootRequestIdValue();
|
|
773
|
+
const responsePromise = new Promise((resolve) => {
|
|
774
|
+
const timer = setTimeout(() => {
|
|
775
|
+
this.pendingTopicRootQueries.delete(requestId);
|
|
776
|
+
resolve(undefined);
|
|
777
|
+
}, DEFAULT_TOPIC_ROOT_QUERY_TIMEOUT_MS);
|
|
778
|
+
timer.unref?.();
|
|
779
|
+
this.pendingTopicRootQueries.set(requestId, { topic, resolve, timer });
|
|
780
|
+
});
|
|
781
|
+
try {
|
|
782
|
+
await this.sendDirectControlMessage(peer, new TopicRootQuery({ requestId, topic }));
|
|
783
|
+
}
|
|
784
|
+
catch {
|
|
785
|
+
const pending = this.pendingTopicRootQueries.get(requestId);
|
|
786
|
+
if (pending) {
|
|
787
|
+
this.pendingTopicRootQueries.delete(requestId);
|
|
788
|
+
clearTimeout(pending.timer);
|
|
789
|
+
pending.resolve(undefined);
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
return responsePromise;
|
|
793
|
+
}
|
|
794
|
+
async resolveTopicRootThroughPeers(topic) {
|
|
795
|
+
const peers = this.getConnectedTopicRootTrackers();
|
|
796
|
+
if (peers.length === 0) {
|
|
797
|
+
return undefined;
|
|
798
|
+
}
|
|
799
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
800
|
+
for (const peer of peers) {
|
|
801
|
+
const resolved = await this.queryTopicRootFromPeer(peer, topic);
|
|
802
|
+
if (resolved) {
|
|
803
|
+
return resolved;
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
if (attempt < 2) {
|
|
807
|
+
await delay(150 * (attempt + 1));
|
|
808
|
+
}
|
|
681
809
|
}
|
|
682
|
-
|
|
683
|
-
return resolved;
|
|
810
|
+
return undefined;
|
|
684
811
|
}
|
|
685
812
|
async ensureFanoutChannel(shardTopic, options) {
|
|
686
813
|
const t = shardTopic.toString();
|
|
@@ -1101,7 +1228,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1101
1228
|
await this.ensureFanoutChannel(shardTopic, { ephemeral: true });
|
|
1102
1229
|
const st = this.fanoutChannels.get(shardTopic);
|
|
1103
1230
|
if (st) {
|
|
1104
|
-
void st.channel.
|
|
1231
|
+
void st.channel.publishMaybe(toUint8Array(embedded.bytes()));
|
|
1105
1232
|
this.touchFanoutChannel(shardTopic);
|
|
1106
1233
|
}
|
|
1107
1234
|
}
|
|
@@ -1128,7 +1255,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1128
1255
|
await this.ensureFanoutChannel(shardTopic, { ephemeral: true });
|
|
1129
1256
|
const st = this.fanoutChannels.get(shardTopic);
|
|
1130
1257
|
if (st) {
|
|
1131
|
-
void st.channel.
|
|
1258
|
+
void st.channel.publishMaybe(toUint8Array(embedded.bytes()));
|
|
1132
1259
|
this.touchFanoutChannel(shardTopic);
|
|
1133
1260
|
}
|
|
1134
1261
|
}
|
|
@@ -1196,9 +1323,14 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1196
1323
|
byShard.set(shardTopic, [...(byShard.get(shardTopic) ?? []), t]);
|
|
1197
1324
|
}
|
|
1198
1325
|
await Promise.all([...byShard.entries()].map(async ([shardTopic, userTopics]) => {
|
|
1326
|
+
const msg = new GetSubscribers({ topics: userTopics });
|
|
1327
|
+
const directPeer = to ? this.peers.get(to.hashcode()) : undefined;
|
|
1328
|
+
if (directPeer) {
|
|
1329
|
+
await this.sendDirectControlMessage(directPeer, msg);
|
|
1330
|
+
return;
|
|
1331
|
+
}
|
|
1199
1332
|
const persistent = (this.shardRefCounts.get(shardTopic) ?? 0) > 0;
|
|
1200
1333
|
await this.ensureFanoutChannel(shardTopic, { ephemeral: !persistent });
|
|
1201
|
-
const msg = new GetSubscribers({ topics: userTopics });
|
|
1202
1334
|
const embedded = await this.createMessage(toUint8Array(msg.bytes()), {
|
|
1203
1335
|
mode: new AnyWhere(),
|
|
1204
1336
|
priority: 1,
|
|
@@ -1215,7 +1347,7 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1215
1347
|
});
|
|
1216
1348
|
}
|
|
1217
1349
|
catch {
|
|
1218
|
-
await st.channel.
|
|
1350
|
+
await st.channel.publishMaybe(payload);
|
|
1219
1351
|
}
|
|
1220
1352
|
}
|
|
1221
1353
|
else {
|
|
@@ -1482,20 +1614,86 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1482
1614
|
catch {
|
|
1483
1615
|
// ignore and fall back
|
|
1484
1616
|
}
|
|
1485
|
-
|
|
1486
|
-
await st.channel.publish(payload);
|
|
1487
|
-
}
|
|
1488
|
-
catch {
|
|
1489
|
-
// ignore
|
|
1490
|
-
}
|
|
1617
|
+
await st.channel.publishMaybe(payload);
|
|
1491
1618
|
}
|
|
1492
1619
|
async processDirectPubSubMessage(input) {
|
|
1493
|
-
const { pubsubMessage, message } = input;
|
|
1620
|
+
const { pubsubMessage, message, from, stream } = input;
|
|
1494
1621
|
if (pubsubMessage instanceof TopicRootCandidates) {
|
|
1495
1622
|
// Used only to converge deterministic shard-root candidates in auto mode.
|
|
1496
1623
|
this.mergeAutoTopicRootCandidatesFromPeer(pubsubMessage.candidates);
|
|
1497
1624
|
return;
|
|
1498
1625
|
}
|
|
1626
|
+
if (pubsubMessage instanceof TopicRootQuery) {
|
|
1627
|
+
const root = await this.topicRootControlPlane.resolveCanonicalTopicRoot(pubsubMessage.topic);
|
|
1628
|
+
await this.sendDirectControlMessage(stream, new TopicRootQueryResponse({
|
|
1629
|
+
requestId: pubsubMessage.requestId,
|
|
1630
|
+
topic: pubsubMessage.topic,
|
|
1631
|
+
root,
|
|
1632
|
+
})).catch(() => { });
|
|
1633
|
+
return;
|
|
1634
|
+
}
|
|
1635
|
+
if (pubsubMessage instanceof TopicRootQueryResponse) {
|
|
1636
|
+
this.resolvePendingTopicRootQuery(pubsubMessage);
|
|
1637
|
+
return;
|
|
1638
|
+
}
|
|
1639
|
+
if (pubsubMessage instanceof Subscribe) {
|
|
1640
|
+
const sender = from;
|
|
1641
|
+
const senderKey = sender.hashcode();
|
|
1642
|
+
const relevantTopics = pubsubMessage.topics.filter((t) => this.isTrackedTopic(t));
|
|
1643
|
+
if (relevantTopics.length > 0 &&
|
|
1644
|
+
this.subscriptionMessageIsLatest(message, pubsubMessage, relevantTopics)) {
|
|
1645
|
+
const changed = [];
|
|
1646
|
+
for (const topic of relevantTopics) {
|
|
1647
|
+
const peers = this.topics.get(topic);
|
|
1648
|
+
if (!peers)
|
|
1649
|
+
continue;
|
|
1650
|
+
this.initializePeer(sender);
|
|
1651
|
+
const existing = peers.get(senderKey);
|
|
1652
|
+
if (!existing || existing.session < message.header.session) {
|
|
1653
|
+
peers.delete(senderKey);
|
|
1654
|
+
peers.set(senderKey, new SubscriptionData({
|
|
1655
|
+
session: message.header.session,
|
|
1656
|
+
timestamp: message.header.timestamp,
|
|
1657
|
+
publicKey: sender,
|
|
1658
|
+
}));
|
|
1659
|
+
changed.push(topic);
|
|
1660
|
+
}
|
|
1661
|
+
else {
|
|
1662
|
+
peers.delete(senderKey);
|
|
1663
|
+
peers.set(senderKey, existing);
|
|
1664
|
+
}
|
|
1665
|
+
if (!existing) {
|
|
1666
|
+
this.peerToTopic.get(senderKey).add(topic);
|
|
1667
|
+
}
|
|
1668
|
+
this.pruneTopicSubscribers(topic);
|
|
1669
|
+
}
|
|
1670
|
+
if (changed.length > 0) {
|
|
1671
|
+
this.dispatchEvent(new CustomEvent("subscribe", {
|
|
1672
|
+
detail: new SubscriptionEvent(sender, changed),
|
|
1673
|
+
}));
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
if (pubsubMessage.requestSubscribers) {
|
|
1677
|
+
const overlap = this.getSubscriptionOverlap(pubsubMessage.topics);
|
|
1678
|
+
if (overlap.length > 0) {
|
|
1679
|
+
await this.sendDirectControlMessage(stream, new Subscribe({
|
|
1680
|
+
topics: overlap,
|
|
1681
|
+
requestSubscribers: false,
|
|
1682
|
+
}));
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
return;
|
|
1686
|
+
}
|
|
1687
|
+
if (pubsubMessage instanceof GetSubscribers) {
|
|
1688
|
+
const overlap = this.getSubscriptionOverlap(pubsubMessage.topics);
|
|
1689
|
+
if (overlap.length === 0)
|
|
1690
|
+
return;
|
|
1691
|
+
await this.sendDirectControlMessage(stream, new Subscribe({
|
|
1692
|
+
topics: overlap,
|
|
1693
|
+
requestSubscribers: false,
|
|
1694
|
+
}));
|
|
1695
|
+
return;
|
|
1696
|
+
}
|
|
1499
1697
|
if (pubsubMessage instanceof PubSubData) {
|
|
1500
1698
|
this.dispatchEvent(new CustomEvent("data", {
|
|
1501
1699
|
detail: new DataEvent({
|
|
@@ -1680,10 +1878,14 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1680
1878
|
catch {
|
|
1681
1879
|
return super.onDataMessage(from, stream, message, seenBefore);
|
|
1682
1880
|
}
|
|
1683
|
-
// DirectStream
|
|
1684
|
-
//
|
|
1881
|
+
// DirectStream supports targeted pubsub data plus targeted subscriber snapshot
|
|
1882
|
+
// traffic used by Program.waitFor() when a peer is already connected directly.
|
|
1685
1883
|
if (!(pubsubMessage instanceof PubSubData) &&
|
|
1686
|
-
!(pubsubMessage instanceof TopicRootCandidates)
|
|
1884
|
+
!(pubsubMessage instanceof TopicRootCandidates) &&
|
|
1885
|
+
!(pubsubMessage instanceof TopicRootQuery) &&
|
|
1886
|
+
!(pubsubMessage instanceof GetSubscribers) &&
|
|
1887
|
+
!(pubsubMessage instanceof Subscribe) &&
|
|
1888
|
+
!(pubsubMessage instanceof TopicRootQueryResponse)) {
|
|
1687
1889
|
return true;
|
|
1688
1890
|
}
|
|
1689
1891
|
// Determine if this node should process it.
|
|
@@ -1704,7 +1906,12 @@ export class TopicControlPlane extends DirectStream {
|
|
|
1704
1906
|
return false;
|
|
1705
1907
|
await this.maybeAcknowledgeMessage(stream, message, seenBefore);
|
|
1706
1908
|
if (seenBefore === 0) {
|
|
1707
|
-
await this.processDirectPubSubMessage({
|
|
1909
|
+
await this.processDirectPubSubMessage({
|
|
1910
|
+
pubsubMessage,
|
|
1911
|
+
message,
|
|
1912
|
+
from,
|
|
1913
|
+
stream,
|
|
1914
|
+
});
|
|
1708
1915
|
}
|
|
1709
1916
|
}
|
|
1710
1917
|
// Forward direct PubSubData only (subscription control lives on fanout shards).
|