@peerbit/pubsub 5.2.11 → 5.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.d.ts +3 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.d.ts.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.js +90 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.js.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-eval.d.ts +2 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-eval.d.ts.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-eval.js +712 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-eval.js.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.d.ts +2 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.d.ts.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.js +1591 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.js.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-prepush.d.ts +2 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-prepush.d.ts.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-prepush.js +297 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-prepush.js.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-preset.d.ts +73 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-preset.d.ts.map +1 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-preset.js +183 -0
- package/dist/benchmark/fanout-tree-parent-upgrade-preset.js.map +1 -0
- package/dist/benchmark/fanout-tree-sim-lib.d.ts +101 -0
- package/dist/benchmark/fanout-tree-sim-lib.d.ts.map +1 -1
- package/dist/benchmark/fanout-tree-sim-lib.js +540 -38
- package/dist/benchmark/fanout-tree-sim-lib.js.map +1 -1
- package/dist/benchmark/fanout-tree-sim.d.ts +0 -9
- package/dist/benchmark/fanout-tree-sim.d.ts.map +1 -1
- package/dist/benchmark/fanout-tree-sim.js +369 -28
- package/dist/benchmark/fanout-tree-sim.js.map +1 -1
- package/dist/benchmark/index.js +20 -0
- package/dist/benchmark/index.js.map +1 -1
- package/dist/benchmark/pubsub-topic-sim-lib.d.ts +4 -0
- package/dist/benchmark/pubsub-topic-sim-lib.d.ts.map +1 -1
- package/dist/benchmark/pubsub-topic-sim-lib.js +58 -3
- package/dist/benchmark/pubsub-topic-sim-lib.js.map +1 -1
- package/dist/benchmark/pubsub-topic-sim.js +18 -0
- package/dist/benchmark/pubsub-topic-sim.js.map +1 -1
- package/dist/src/debounced-set.d.ts +1 -1
- package/dist/src/debounced-set.d.ts.map +1 -1
- package/dist/src/debounced-set.js +5 -1
- package/dist/src/debounced-set.js.map +1 -1
- package/dist/src/fanout-tree-codec.d.ts +330 -0
- package/dist/src/fanout-tree-codec.d.ts.map +1 -0
- package/dist/src/fanout-tree-codec.js +1209 -0
- package/dist/src/fanout-tree-codec.js.map +1 -0
- package/dist/src/fanout-tree-parent-upgrade.d.ts +117 -0
- package/dist/src/fanout-tree-parent-upgrade.d.ts.map +1 -0
- package/dist/src/fanout-tree-parent-upgrade.js +135 -0
- package/dist/src/fanout-tree-parent-upgrade.js.map +1 -0
- package/dist/src/fanout-tree.d.ts +245 -73
- package/dist/src/fanout-tree.d.ts.map +1 -1
- package/dist/src/fanout-tree.js +2290 -1057
- package/dist/src/fanout-tree.js.map +1 -1
- package/dist/src/index.d.ts +58 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +316 -45
- package/dist/src/index.js.map +1 -1
- package/dist/src/topic-root-control-plane.d.ts +13 -0
- package/dist/src/topic-root-control-plane.d.ts.map +1 -1
- package/dist/src/topic-root-control-plane.js +45 -0
- package/dist/src/topic-root-control-plane.js.map +1 -1
- package/package.json +9 -12
- package/src/debounced-set.ts +5 -0
- package/src/fanout-tree-codec.ts +1580 -0
- package/src/fanout-tree-parent-upgrade.ts +353 -0
- package/src/fanout-tree.ts +4558 -2230
- package/src/index.ts +425 -59
- package/src/topic-root-control-plane.ts +48 -0
package/src/index.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
type DirectStreamComponents,
|
|
29
29
|
type DirectStreamOptions,
|
|
30
30
|
type PeerStreams,
|
|
31
|
+
type RustTopicControl,
|
|
31
32
|
} from "@peerbit/stream";
|
|
32
33
|
import {
|
|
33
34
|
AcknowledgeAnyWhere,
|
|
@@ -64,12 +65,30 @@ import type {
|
|
|
64
65
|
import { TopicRootControlPlane } from "./topic-root-control-plane.js";
|
|
65
66
|
|
|
66
67
|
export * from "./fanout-tree.js";
|
|
68
|
+
// The complete /peerbit/fanout-tree/0.5.0 wire codec and the
|
|
69
|
+
// parent-upgrade decision module (the TS references the native rust-core
|
|
70
|
+
// implementations are parity-tested against).
|
|
71
|
+
export * as fanoutWire from "./fanout-tree-codec.js";
|
|
72
|
+
export * as fanoutParentUpgrade from "./fanout-tree-parent-upgrade.js";
|
|
67
73
|
export * from "./fanout-channel.js";
|
|
68
74
|
export * from "./topic-root-control-plane.js";
|
|
69
75
|
export { toUint8Array } from "./bytes.js";
|
|
70
76
|
|
|
71
77
|
export const logger = loggerFn("peerbit:transport:topic-control-plane");
|
|
72
78
|
const warn = logger.newScope("warn");
|
|
79
|
+
|
|
80
|
+
const utf8Encoder = new TextEncoder();
|
|
81
|
+
/**
|
|
82
|
+
* Offset of the `data` bytes inside a borsh-serialized `PubSubData`
|
|
83
|
+
* (variant byte + topics vec + strict bool + data length prefix).
|
|
84
|
+
*/
|
|
85
|
+
const preEncodedPubSubDataOffset = (topics: string[]): number => {
|
|
86
|
+
let offset = 1 + 4; // variant + topics vec length
|
|
87
|
+
for (const topic of topics) {
|
|
88
|
+
offset += 4 + utf8Encoder.encode(topic).byteLength;
|
|
89
|
+
}
|
|
90
|
+
return offset + 1 + 4; // strict + data length prefix
|
|
91
|
+
};
|
|
73
92
|
const logError = (e?: { message: string }) => {
|
|
74
93
|
logger.error(e?.message);
|
|
75
94
|
};
|
|
@@ -119,6 +138,7 @@ const AUTO_TOPIC_ROOT_CANDIDATES_MAX = 64;
|
|
|
119
138
|
// Topic-root queries may need to wait for the responder to finish opening an
|
|
120
139
|
// outbound stream back to the requester after an inbound-only dial.
|
|
121
140
|
const DEFAULT_TOPIC_ROOT_QUERY_TIMEOUT_MS = 12_000;
|
|
141
|
+
const DIRECT_SHARD_ROOT_CONFIRM_TIMEOUT_MS = 2_000;
|
|
122
142
|
|
|
123
143
|
const DEFAULT_PUBSUB_FANOUT_CHANNEL_OPTIONS: Omit<
|
|
124
144
|
FanoutTreeChannelOptions,
|
|
@@ -238,6 +258,21 @@ export class TopicControlPlane
|
|
|
238
258
|
public subscriptions: Map<string, { counter: number }>;
|
|
239
259
|
// Local topics requested via debounced subscribe, not yet applied in `subscriptions`.
|
|
240
260
|
private pendingSubscriptions: Set<string>;
|
|
261
|
+
/**
|
|
262
|
+
* Per-shard batching of Subscribe{requestSubscribers} responses. When many
|
|
263
|
+
* peers announce subscriptions in a burst (group subscribe), answering each
|
|
264
|
+
* announcer with a routed unicast is quadratic in subscriber count and
|
|
265
|
+
* melts the fanout route-resolution machinery (#983); a short window lets
|
|
266
|
+
* one broadcast answer all concurrent announcers instead.
|
|
267
|
+
*/
|
|
268
|
+
private pendingSubscriberResponses: Map<
|
|
269
|
+
string,
|
|
270
|
+
{
|
|
271
|
+
targets: Set<string>;
|
|
272
|
+
topics: Set<string>;
|
|
273
|
+
timer: ReturnType<typeof setTimeout>;
|
|
274
|
+
}
|
|
275
|
+
> = new Map();
|
|
241
276
|
public lastSubscriptionMessages: Map<
|
|
242
277
|
string,
|
|
243
278
|
Map<string, { session: bigint; timestamp: bigint }>
|
|
@@ -246,6 +281,13 @@ export class TopicControlPlane
|
|
|
246
281
|
public readonly topicRootControlPlane: TopicRootControlPlane;
|
|
247
282
|
public readonly subscriberCacheMaxEntries: number;
|
|
248
283
|
public readonly fanout: FanoutTree;
|
|
284
|
+
// Native topic-control components of the rust-core DirectStream engine
|
|
285
|
+
// (`@peerbit/network-rust`). When set, the PubSubMessage codec, the topic
|
|
286
|
+
// hashing behind shard mapping and root selection, the root-directory
|
|
287
|
+
// state and the subscribe-state convergence rules run natively; the
|
|
288
|
+
// observable subscription maps and all events stay unchanged. Unset (the
|
|
289
|
+
// default) leaves every code path as-is.
|
|
290
|
+
private readonly nativeTopicControl?: RustTopicControl;
|
|
249
291
|
|
|
250
292
|
private debounceSubscribeAggregator: DebouncedAccumulatorCounterMap;
|
|
251
293
|
private debounceUnsubscribeAggregator: DebouncedAccumulatorCounterMap;
|
|
@@ -253,10 +295,7 @@ export class TopicControlPlane
|
|
|
253
295
|
private readonly shardCount: number;
|
|
254
296
|
private readonly shardTopicPrefix: string;
|
|
255
297
|
private readonly hostShards: boolean;
|
|
256
|
-
private readonly shardRootCache = new Map<
|
|
257
|
-
string,
|
|
258
|
-
{ root: string; authoritative: boolean }
|
|
259
|
-
>();
|
|
298
|
+
private readonly shardRootCache = new Map<string, { root: string; authoritative: boolean }>();
|
|
260
299
|
private readonly shardTopicCache = new Map<string, string>();
|
|
261
300
|
private readonly shardRefCounts = new Map<string, number>();
|
|
262
301
|
private readonly pinnedShards = new Set<string>();
|
|
@@ -325,6 +364,12 @@ export class TopicControlPlane
|
|
|
325
364
|
|
|
326
365
|
this.topicRootControlPlane =
|
|
327
366
|
props?.topicRootControlPlane || new TopicRootControlPlane();
|
|
367
|
+
this.nativeTopicControl = this.rustCore?.topicControl;
|
|
368
|
+
if (this.nativeTopicControl) {
|
|
369
|
+
this.topicRootControlPlane.adoptNativeDirectoryState(
|
|
370
|
+
this.nativeTopicControl.createRootDirectoryState(),
|
|
371
|
+
);
|
|
372
|
+
}
|
|
328
373
|
this.dispatchEventOnSelfPublish =
|
|
329
374
|
props?.dispatchEventOnSelfPublish || false;
|
|
330
375
|
|
|
@@ -393,12 +438,18 @@ export class TopicControlPlane
|
|
|
393
438
|
this.debounceSubscribeAggregator = debouncedAccumulatorSetCounter(
|
|
394
439
|
(set) => this._subscribe([...set.values()]),
|
|
395
440
|
props?.subscriptionDebounceDelay ?? 50,
|
|
441
|
+
(error) =>
|
|
442
|
+
logger.error(`Debounced subscribe failed: ${error?.message ?? error}`),
|
|
396
443
|
);
|
|
397
444
|
// NOTE: Unsubscribe should update local state immediately and batch only the
|
|
398
445
|
// best-effort network announcements to avoid teardown stalls (program close).
|
|
399
446
|
this.debounceUnsubscribeAggregator = debouncedAccumulatorSetCounter(
|
|
400
447
|
(set) => this._announceUnsubscribe([...set.values()]),
|
|
401
448
|
props?.subscriptionDebounceDelay ?? 50,
|
|
449
|
+
(error) =>
|
|
450
|
+
logger.error(
|
|
451
|
+
`Debounced unsubscribe announce failed: ${error?.message ?? error}`,
|
|
452
|
+
),
|
|
402
453
|
);
|
|
403
454
|
}
|
|
404
455
|
|
|
@@ -497,6 +548,10 @@ export class TopicControlPlane
|
|
|
497
548
|
|
|
498
549
|
this.subscriptions.clear();
|
|
499
550
|
this.pendingSubscriptions.clear();
|
|
551
|
+
for (const entry of this.pendingSubscriberResponses.values()) {
|
|
552
|
+
clearTimeout(entry.timer);
|
|
553
|
+
}
|
|
554
|
+
this.pendingSubscriberResponses.clear();
|
|
500
555
|
this.topics.clear();
|
|
501
556
|
this.peerToTopic.clear();
|
|
502
557
|
this.lastSubscriptionMessages.clear();
|
|
@@ -611,6 +666,12 @@ export class TopicControlPlane
|
|
|
611
666
|
}
|
|
612
667
|
|
|
613
668
|
private normalizeAutoTopicRootCandidates(candidates: string[]): string[] {
|
|
669
|
+
if (this.nativeTopicControl) {
|
|
670
|
+
return this.nativeTopicControl.normalizeAutoCandidates(
|
|
671
|
+
candidates,
|
|
672
|
+
this.publicKeyHash,
|
|
673
|
+
);
|
|
674
|
+
}
|
|
614
675
|
const unique = new Set<string>();
|
|
615
676
|
for (const c of candidates) {
|
|
616
677
|
if (!c) continue;
|
|
@@ -680,7 +741,7 @@ export class TopicControlPlane
|
|
|
680
741
|
if (candidates.length === 0) return;
|
|
681
742
|
|
|
682
743
|
const msg = new TopicRootCandidates({ candidates });
|
|
683
|
-
const embedded = await this.createMessage(
|
|
744
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
|
|
684
745
|
mode: new AnyWhere(),
|
|
685
746
|
priority: 1,
|
|
686
747
|
skipRecipientValidation: true,
|
|
@@ -776,7 +837,7 @@ export class TopicControlPlane
|
|
|
776
837
|
topics: userTopics,
|
|
777
838
|
requestSubscribers: true,
|
|
778
839
|
});
|
|
779
|
-
const embedded = await this.createMessage(
|
|
840
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
|
|
780
841
|
mode: new AnyWhere(),
|
|
781
842
|
priority: 1,
|
|
782
843
|
skipRecipientValidation: true,
|
|
@@ -925,12 +986,118 @@ export class TopicControlPlane
|
|
|
925
986
|
const t = topic.toString();
|
|
926
987
|
const cached = this.shardTopicCache.get(t);
|
|
927
988
|
if (cached) return cached;
|
|
928
|
-
const
|
|
929
|
-
|
|
989
|
+
const shardTopic = this.nativeTopicControl
|
|
990
|
+
? this.nativeTopicControl.shardTopic(
|
|
991
|
+
t,
|
|
992
|
+
this.shardCount,
|
|
993
|
+
this.shardTopicPrefix,
|
|
994
|
+
)
|
|
995
|
+
: `${this.shardTopicPrefix}${topicHash32(t) % this.shardCount}`;
|
|
930
996
|
this.shardTopicCache.set(t, shardTopic);
|
|
931
997
|
return shardTopic;
|
|
932
998
|
}
|
|
933
999
|
|
|
1000
|
+
/**
|
|
1001
|
+
* Serialize a control-plane message (borsh `PubSubMessage`); rust-core
|
|
1002
|
+
* mode encodes natively (byte-identical, covered by the parity suite).
|
|
1003
|
+
*/
|
|
1004
|
+
private encodePubSubMessage(message: PubSubMessage): Uint8Array {
|
|
1005
|
+
const native = this.nativeTopicControl;
|
|
1006
|
+
if (native) {
|
|
1007
|
+
if (message instanceof PubSubData) {
|
|
1008
|
+
return native.encodePubSubData(
|
|
1009
|
+
message.topics,
|
|
1010
|
+
message.strict,
|
|
1011
|
+
message.data,
|
|
1012
|
+
);
|
|
1013
|
+
}
|
|
1014
|
+
if (message instanceof Subscribe) {
|
|
1015
|
+
return native.encodeSubscribe(
|
|
1016
|
+
message.topics,
|
|
1017
|
+
message.requestSubscribers,
|
|
1018
|
+
);
|
|
1019
|
+
}
|
|
1020
|
+
if (message instanceof Unsubscribe) {
|
|
1021
|
+
return native.encodeUnsubscribe(message.topics);
|
|
1022
|
+
}
|
|
1023
|
+
if (message instanceof GetSubscribers) {
|
|
1024
|
+
return native.encodeGetSubscribers(message.topics);
|
|
1025
|
+
}
|
|
1026
|
+
if (message instanceof TopicRootCandidates) {
|
|
1027
|
+
return native.encodeTopicRootCandidates(message.candidates);
|
|
1028
|
+
}
|
|
1029
|
+
if (message instanceof PeerUnavailable) {
|
|
1030
|
+
return native.encodePeerUnavailable(
|
|
1031
|
+
message.publicKeyHash,
|
|
1032
|
+
message.session,
|
|
1033
|
+
message.timestamp,
|
|
1034
|
+
message.topics,
|
|
1035
|
+
);
|
|
1036
|
+
}
|
|
1037
|
+
if (message instanceof TopicRootQuery) {
|
|
1038
|
+
return native.encodeTopicRootQuery(message.requestId, message.topic);
|
|
1039
|
+
}
|
|
1040
|
+
if (message instanceof TopicRootQueryResponse) {
|
|
1041
|
+
return native.encodeTopicRootQueryResponse(
|
|
1042
|
+
message.requestId,
|
|
1043
|
+
message.topic,
|
|
1044
|
+
message.root,
|
|
1045
|
+
);
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
return toUint8Array(message.bytes());
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
/**
|
|
1052
|
+
* Parse a control-plane payload; rust-core mode decodes natively into the
|
|
1053
|
+
* same message classes (decode failures throw either way, so callers keep
|
|
1054
|
+
* their fallback behavior).
|
|
1055
|
+
*/
|
|
1056
|
+
private decodePubSubMessage(bytes: Uint8Array): PubSubMessage {
|
|
1057
|
+
const native = this.nativeTopicControl;
|
|
1058
|
+
if (native) {
|
|
1059
|
+
const decoded = native.decodePubSubMessage(bytes);
|
|
1060
|
+
switch (decoded.type) {
|
|
1061
|
+
case "data":
|
|
1062
|
+
return new PubSubData({
|
|
1063
|
+
topics: decoded.topics,
|
|
1064
|
+
data: decoded.data,
|
|
1065
|
+
strict: decoded.strict,
|
|
1066
|
+
});
|
|
1067
|
+
case "subscribe":
|
|
1068
|
+
return new Subscribe({
|
|
1069
|
+
topics: decoded.topics,
|
|
1070
|
+
requestSubscribers: decoded.requestSubscribers,
|
|
1071
|
+
});
|
|
1072
|
+
case "unsubscribe":
|
|
1073
|
+
return new Unsubscribe({ topics: decoded.topics });
|
|
1074
|
+
case "get-subscribers":
|
|
1075
|
+
return new GetSubscribers({ topics: decoded.topics });
|
|
1076
|
+
case "topic-root-candidates":
|
|
1077
|
+
return new TopicRootCandidates({ candidates: decoded.candidates });
|
|
1078
|
+
case "peer-unavailable":
|
|
1079
|
+
return new PeerUnavailable({
|
|
1080
|
+
publicKeyHash: decoded.publicKeyHash,
|
|
1081
|
+
session: decoded.session,
|
|
1082
|
+
timestamp: decoded.timestamp,
|
|
1083
|
+
topics: decoded.topics,
|
|
1084
|
+
});
|
|
1085
|
+
case "topic-root-query":
|
|
1086
|
+
return new TopicRootQuery({
|
|
1087
|
+
requestId: decoded.requestId,
|
|
1088
|
+
topic: decoded.topic,
|
|
1089
|
+
});
|
|
1090
|
+
case "topic-root-query-response":
|
|
1091
|
+
return new TopicRootQueryResponse({
|
|
1092
|
+
requestId: decoded.requestId,
|
|
1093
|
+
topic: decoded.topic,
|
|
1094
|
+
root: decoded.root,
|
|
1095
|
+
});
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
return PubSubMessage.from(bytes);
|
|
1099
|
+
}
|
|
1100
|
+
|
|
934
1101
|
private async resolveTopicRootState(
|
|
935
1102
|
topic: string,
|
|
936
1103
|
): Promise<{ root?: string; authoritative: boolean }> {
|
|
@@ -1024,7 +1191,7 @@ export class TopicControlPlane
|
|
|
1024
1191
|
peer: PeerStreams,
|
|
1025
1192
|
pubsubMessage: PubSubMessage,
|
|
1026
1193
|
) {
|
|
1027
|
-
const embedded = await this.createMessage(
|
|
1194
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(pubsubMessage), {
|
|
1028
1195
|
mode: new SilentDelivery({
|
|
1029
1196
|
to: [peer.publicKey.hashcode()],
|
|
1030
1197
|
redundancy: 1,
|
|
@@ -1051,6 +1218,7 @@ export class TopicControlPlane
|
|
|
1051
1218
|
private async queryTopicRootFromPeer(
|
|
1052
1219
|
peer: PeerStreams,
|
|
1053
1220
|
topic: string,
|
|
1221
|
+
timeoutMs = DEFAULT_TOPIC_ROOT_QUERY_TIMEOUT_MS,
|
|
1054
1222
|
): Promise<string | undefined> {
|
|
1055
1223
|
if (!this.started || this.stopping) return undefined;
|
|
1056
1224
|
|
|
@@ -1059,7 +1227,7 @@ export class TopicControlPlane
|
|
|
1059
1227
|
const timer = setTimeout(() => {
|
|
1060
1228
|
this.pendingTopicRootQueries.delete(requestId);
|
|
1061
1229
|
resolve(undefined);
|
|
1062
|
-
},
|
|
1230
|
+
}, Math.max(1, Math.floor(timeoutMs)));
|
|
1063
1231
|
timer.unref?.();
|
|
1064
1232
|
this.pendingTopicRootQueries.set(requestId, { topic, resolve, timer });
|
|
1065
1233
|
});
|
|
@@ -1081,6 +1249,33 @@ export class TopicControlPlane
|
|
|
1081
1249
|
return responsePromise;
|
|
1082
1250
|
}
|
|
1083
1251
|
|
|
1252
|
+
private async confirmDirectShardRoot(
|
|
1253
|
+
shardTopic: string,
|
|
1254
|
+
root: string,
|
|
1255
|
+
signal?: AbortSignal,
|
|
1256
|
+
): Promise<string> {
|
|
1257
|
+
if (root === this.publicKeyHash) return root;
|
|
1258
|
+
const rootPeer = this.peers.get(root);
|
|
1259
|
+
if (!rootPeer) return root;
|
|
1260
|
+
|
|
1261
|
+
const confirmation = await withAbort(
|
|
1262
|
+
this.queryTopicRootFromPeer(
|
|
1263
|
+
rootPeer,
|
|
1264
|
+
shardTopic,
|
|
1265
|
+
DIRECT_SHARD_ROOT_CONFIRM_TIMEOUT_MS,
|
|
1266
|
+
),
|
|
1267
|
+
signal,
|
|
1268
|
+
);
|
|
1269
|
+
if (!confirmation) return root;
|
|
1270
|
+
if (confirmation !== root) {
|
|
1271
|
+
this.shardRootCache.set(shardTopic, {
|
|
1272
|
+
root: confirmation,
|
|
1273
|
+
authoritative: true,
|
|
1274
|
+
});
|
|
1275
|
+
}
|
|
1276
|
+
return confirmation;
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1084
1279
|
private async resolveQueryableTopicRoot(
|
|
1085
1280
|
topic: string,
|
|
1086
1281
|
): Promise<string | undefined> {
|
|
@@ -1137,6 +1332,7 @@ export class TopicControlPlane
|
|
|
1137
1332
|
signal?: AbortSignal;
|
|
1138
1333
|
},
|
|
1139
1334
|
): Promise<void> {
|
|
1335
|
+
if (!this.started) throw new NotStartedError();
|
|
1140
1336
|
const t = shardTopic.toString();
|
|
1141
1337
|
const pin = options?.pin === true;
|
|
1142
1338
|
const wantEphemeral = options?.ephemeral === true;
|
|
@@ -1166,6 +1362,7 @@ export class TopicControlPlane
|
|
|
1166
1362
|
}
|
|
1167
1363
|
|
|
1168
1364
|
root = root ?? (await this.resolveShardRoot(t));
|
|
1365
|
+
root = await this.confirmDirectShardRoot(t, root, options?.signal);
|
|
1169
1366
|
const channel = new FanoutChannel(this.fanout, { topic: t, root });
|
|
1170
1367
|
|
|
1171
1368
|
const onPayload = (payload: Uint8Array) => {
|
|
@@ -1186,7 +1383,7 @@ export class TopicControlPlane
|
|
|
1186
1383
|
|
|
1187
1384
|
let pubsubMessage: PubSubMessage;
|
|
1188
1385
|
try {
|
|
1189
|
-
pubsubMessage =
|
|
1386
|
+
pubsubMessage = this.decodePubSubMessage(dm.data);
|
|
1190
1387
|
} catch {
|
|
1191
1388
|
return;
|
|
1192
1389
|
}
|
|
@@ -1232,6 +1429,9 @@ export class TopicControlPlane
|
|
|
1232
1429
|
this.seenCache.add(msgId, seen ? seen + 1 : 1);
|
|
1233
1430
|
if (seen) return;
|
|
1234
1431
|
|
|
1432
|
+
// fanout-delivered frames bypass the inbound stream decode, so
|
|
1433
|
+
// the nested envelope is verified natively here when possible
|
|
1434
|
+
this.seedNativeWireVerification(dm, payload);
|
|
1235
1435
|
if ((await this.verifyAndProcess(dm)) === false) {
|
|
1236
1436
|
return;
|
|
1237
1437
|
}
|
|
@@ -1295,9 +1495,9 @@ export class TopicControlPlane
|
|
|
1295
1495
|
// ignore
|
|
1296
1496
|
}
|
|
1297
1497
|
try {
|
|
1298
|
-
channel.
|
|
1498
|
+
await channel.leave({ notifyParent: false, kickChildren: false });
|
|
1299
1499
|
} catch {
|
|
1300
|
-
|
|
1500
|
+
channel.close();
|
|
1301
1501
|
}
|
|
1302
1502
|
throw error;
|
|
1303
1503
|
}
|
|
@@ -1419,7 +1619,7 @@ export class TopicControlPlane
|
|
|
1419
1619
|
topics: userTopics,
|
|
1420
1620
|
requestSubscribers: true,
|
|
1421
1621
|
});
|
|
1422
|
-
const embedded = await this.createMessage(
|
|
1622
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
|
|
1423
1623
|
mode: new AnyWhere(),
|
|
1424
1624
|
priority: 1,
|
|
1425
1625
|
skipRecipientValidation: true,
|
|
@@ -1498,7 +1698,7 @@ export class TopicControlPlane
|
|
|
1498
1698
|
// Announce first.
|
|
1499
1699
|
try {
|
|
1500
1700
|
const msg = new Unsubscribe({ topics: userTopics });
|
|
1501
|
-
const embedded = await this.createMessage(
|
|
1701
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
|
|
1502
1702
|
mode: new AnyWhere(),
|
|
1503
1703
|
priority: 1,
|
|
1504
1704
|
skipRecipientValidation: true,
|
|
@@ -1571,7 +1771,7 @@ export class TopicControlPlane
|
|
|
1571
1771
|
timestamp: batch.timestamp,
|
|
1572
1772
|
topics: batch.topics,
|
|
1573
1773
|
});
|
|
1574
|
-
const embedded = await this.createMessage(
|
|
1774
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
|
|
1575
1775
|
mode: new AnyWhere(),
|
|
1576
1776
|
priority: 1,
|
|
1577
1777
|
skipRecipientValidation: true,
|
|
@@ -1601,7 +1801,7 @@ export class TopicControlPlane
|
|
|
1601
1801
|
timestamp: 0n,
|
|
1602
1802
|
topics: [],
|
|
1603
1803
|
});
|
|
1604
|
-
const embedded = await this.createMessage(
|
|
1804
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
|
|
1605
1805
|
mode: new AnyWhere(),
|
|
1606
1806
|
priority: 1,
|
|
1607
1807
|
skipRecipientValidation: true,
|
|
@@ -1702,7 +1902,7 @@ export class TopicControlPlane
|
|
|
1702
1902
|
const persistent = (this.shardRefCounts.get(shardTopic) ?? 0) > 0;
|
|
1703
1903
|
await this.ensureFanoutChannel(shardTopic, { ephemeral: !persistent });
|
|
1704
1904
|
|
|
1705
|
-
const embedded = await this.createMessage(
|
|
1905
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
|
|
1706
1906
|
mode: new AnyWhere(),
|
|
1707
1907
|
priority: 1,
|
|
1708
1908
|
skipRecipientValidation: true,
|
|
@@ -1754,10 +1954,13 @@ export class TopicControlPlane
|
|
|
1754
1954
|
const msg = data
|
|
1755
1955
|
? new PubSubData({ topics: topicsAll, data, strict: true })
|
|
1756
1956
|
: undefined;
|
|
1757
|
-
const message = await this.createMessage(
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1957
|
+
const message = await this.createMessage(
|
|
1958
|
+
msg ? this.encodePubSubMessage(msg) : undefined,
|
|
1959
|
+
{
|
|
1960
|
+
...options,
|
|
1961
|
+
skipRecipientValidation: this.dispatchEventOnSelfPublish,
|
|
1962
|
+
},
|
|
1963
|
+
);
|
|
1761
1964
|
|
|
1762
1965
|
if (msg) {
|
|
1763
1966
|
this.dispatchEvent(
|
|
@@ -1800,7 +2003,7 @@ export class TopicControlPlane
|
|
|
1800
2003
|
}
|
|
1801
2004
|
|
|
1802
2005
|
const msg = new PubSubData({ topics: topicsAll, data, strict: false });
|
|
1803
|
-
const embedded = await this.createMessage(
|
|
2006
|
+
const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
|
|
1804
2007
|
mode: new AnyWhere(),
|
|
1805
2008
|
priority: options?.priority,
|
|
1806
2009
|
responsePriority: options?.responsePriority,
|
|
@@ -1862,6 +2065,76 @@ export class TopicControlPlane
|
|
|
1862
2065
|
return embedded.id;
|
|
1863
2066
|
}
|
|
1864
2067
|
|
|
2068
|
+
/**
|
|
2069
|
+
* Publish a pre-encoded `PubSubData` payload to explicit recipients.
|
|
2070
|
+
*
|
|
2071
|
+
* `payload` must be the exact borsh serialization of
|
|
2072
|
+
* `new PubSubData({ topics, data, strict: true })` — callers that encode
|
|
2073
|
+
* the payload elsewhere (e.g. the shared-log fused send path serializing
|
|
2074
|
+
* inside wasm) hand the finished bytes here so the wrapping `PubSubData`
|
|
2075
|
+
* copy of the regular `publish` path is skipped. Everything else matches
|
|
2076
|
+
* the explicit-recipient branch of {@link publish}: the payload becomes a
|
|
2077
|
+
* signed `DataMessage` delivered over DirectStream.
|
|
2078
|
+
*/
|
|
2079
|
+
async publishPreEncodedData(
|
|
2080
|
+
payload: Uint8Array,
|
|
2081
|
+
properties: {
|
|
2082
|
+
topics: string[];
|
|
2083
|
+
},
|
|
2084
|
+
options: {
|
|
2085
|
+
mode: SilentDelivery | AcknowledgeDelivery;
|
|
2086
|
+
} & { client?: string } & PriorityOptions &
|
|
2087
|
+
ResponsePriorityOptions &
|
|
2088
|
+
ExpiresAtOptions &
|
|
2089
|
+
IdOptions &
|
|
2090
|
+
WithExtraSigners & { signal?: AbortSignal },
|
|
2091
|
+
): Promise<Uint8Array | undefined> {
|
|
2092
|
+
if (!this.started) throw new NotStartedError();
|
|
2093
|
+
if (!options?.mode || !deliveryModeHasReceiver(options.mode)) {
|
|
2094
|
+
throw new Error(
|
|
2095
|
+
"publishPreEncodedData requires a delivery mode with explicit recipients",
|
|
2096
|
+
);
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
const message = await this.createMessage(payload, {
|
|
2100
|
+
...options,
|
|
2101
|
+
skipRecipientValidation: this.dispatchEventOnSelfPublish,
|
|
2102
|
+
});
|
|
2103
|
+
|
|
2104
|
+
this.dispatchEvent(
|
|
2105
|
+
new CustomEvent("publish", {
|
|
2106
|
+
detail: new PublishEvent({
|
|
2107
|
+
client: options?.client,
|
|
2108
|
+
data: new PubSubData({
|
|
2109
|
+
topics: properties.topics,
|
|
2110
|
+
data: payload.subarray(
|
|
2111
|
+
preEncodedPubSubDataOffset(properties.topics),
|
|
2112
|
+
),
|
|
2113
|
+
strict: true,
|
|
2114
|
+
}),
|
|
2115
|
+
message,
|
|
2116
|
+
}),
|
|
2117
|
+
}),
|
|
2118
|
+
);
|
|
2119
|
+
|
|
2120
|
+
const silentDelivery = options.mode instanceof SilentDelivery;
|
|
2121
|
+
try {
|
|
2122
|
+
await this.publishMessage(
|
|
2123
|
+
this.publicKey,
|
|
2124
|
+
message,
|
|
2125
|
+
undefined,
|
|
2126
|
+
undefined,
|
|
2127
|
+
options?.signal,
|
|
2128
|
+
);
|
|
2129
|
+
} catch (error) {
|
|
2130
|
+
if (error instanceof DeliveryError && silentDelivery !== false) {
|
|
2131
|
+
return message.id;
|
|
2132
|
+
}
|
|
2133
|
+
throw error;
|
|
2134
|
+
}
|
|
2135
|
+
return message.id;
|
|
2136
|
+
}
|
|
2137
|
+
|
|
1865
2138
|
public onPeerSession(key: PublicSignKey, _session: number): void {
|
|
1866
2139
|
if (!Number.isFinite(_session) || _session < 0) {
|
|
1867
2140
|
return;
|
|
@@ -1999,23 +2272,44 @@ export class TopicControlPlane
|
|
|
1999
2272
|
timestamp: bigint,
|
|
2000
2273
|
relevantTopics: string[],
|
|
2001
2274
|
) {
|
|
2002
|
-
|
|
2003
|
-
const
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2275
|
+
if (this.nativeTopicControl) {
|
|
2276
|
+
const lasts: bigint[] = [];
|
|
2277
|
+
for (const topic of relevantTopics) {
|
|
2278
|
+
const last = this.lastSubscriptionMessages
|
|
2279
|
+
.get(subscriberKey)
|
|
2280
|
+
?.get(topic);
|
|
2281
|
+
if (last) {
|
|
2282
|
+
lasts.push(last.session, last.timestamp);
|
|
2283
|
+
}
|
|
2011
2284
|
}
|
|
2012
2285
|
if (
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2286
|
+
!this.nativeTopicControl.subscriptionIsLatest(
|
|
2287
|
+
BigUint64Array.from(lasts),
|
|
2288
|
+
session,
|
|
2289
|
+
timestamp,
|
|
2290
|
+
)
|
|
2016
2291
|
) {
|
|
2017
2292
|
return false;
|
|
2018
2293
|
}
|
|
2294
|
+
} else {
|
|
2295
|
+
for (const topic of relevantTopics) {
|
|
2296
|
+
const last = this.lastSubscriptionMessages
|
|
2297
|
+
.get(subscriberKey)
|
|
2298
|
+
?.get(topic);
|
|
2299
|
+
if (!last) {
|
|
2300
|
+
continue;
|
|
2301
|
+
}
|
|
2302
|
+
if (last.session > session) {
|
|
2303
|
+
return false;
|
|
2304
|
+
}
|
|
2305
|
+
if (
|
|
2306
|
+
timestamp !== 0n &&
|
|
2307
|
+
last.session === session &&
|
|
2308
|
+
last.timestamp > timestamp
|
|
2309
|
+
) {
|
|
2310
|
+
return false;
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2019
2313
|
}
|
|
2020
2314
|
|
|
2021
2315
|
for (const topic of relevantTopics) {
|
|
@@ -2032,6 +2326,23 @@ export class TopicControlPlane
|
|
|
2032
2326
|
return true;
|
|
2033
2327
|
}
|
|
2034
2328
|
|
|
2329
|
+
/**
|
|
2330
|
+
* `Subscribe` replaces tracked subscription data for new subscribers or
|
|
2331
|
+
* strictly newer sessions; equal/older sessions only refresh cache order.
|
|
2332
|
+
*/
|
|
2333
|
+
private subscribeShouldReplace(
|
|
2334
|
+
existing: SubscriptionData | undefined,
|
|
2335
|
+
session: bigint,
|
|
2336
|
+
): boolean {
|
|
2337
|
+
if (this.nativeTopicControl) {
|
|
2338
|
+
return this.nativeTopicControl.subscribeShouldReplace(
|
|
2339
|
+
existing?.session,
|
|
2340
|
+
session,
|
|
2341
|
+
);
|
|
2342
|
+
}
|
|
2343
|
+
return !existing || existing.session < session;
|
|
2344
|
+
}
|
|
2345
|
+
|
|
2035
2346
|
private subscriptionMessageIsLatest(
|
|
2036
2347
|
message: DataMessage,
|
|
2037
2348
|
_pubsubMessage: Subscribe | Unsubscribe,
|
|
@@ -2046,6 +2357,78 @@ export class TopicControlPlane
|
|
|
2046
2357
|
);
|
|
2047
2358
|
}
|
|
2048
2359
|
|
|
2360
|
+
/**
|
|
2361
|
+
* Answer Subscribe{requestSubscribers} announces without letting bursts go
|
|
2362
|
+
* quadratic (#983). Leading edge: the first announcer is answered
|
|
2363
|
+
* immediately with a targeted unicast — a lone joiner sees no added
|
|
2364
|
+
* latency. Announcers arriving within the trailing window that this opens
|
|
2365
|
+
* are batched and answered with ONE broadcast, instead of S(S-1) routed
|
|
2366
|
+
* unicasts whose cold route resolutions flood the shard tree.
|
|
2367
|
+
*/
|
|
2368
|
+
private queueSubscriberResponse(
|
|
2369
|
+
shardTopic: string,
|
|
2370
|
+
targetHash: string,
|
|
2371
|
+
topics: string[],
|
|
2372
|
+
) {
|
|
2373
|
+
let entry = this.pendingSubscriberResponses.get(shardTopic);
|
|
2374
|
+
if (!entry) {
|
|
2375
|
+
const created = {
|
|
2376
|
+
targets: new Set<string>(),
|
|
2377
|
+
topics: new Set<string>(),
|
|
2378
|
+
timer: setTimeout(
|
|
2379
|
+
() => {
|
|
2380
|
+
this.pendingSubscriberResponses.delete(shardTopic);
|
|
2381
|
+
if (created.targets.size === 0) return;
|
|
2382
|
+
void this.flushSubscriberResponses(shardTopic, created).catch(
|
|
2383
|
+
logErrorIfStarted,
|
|
2384
|
+
);
|
|
2385
|
+
},
|
|
2386
|
+
150 + Math.floor(Math.random() * 150),
|
|
2387
|
+
),
|
|
2388
|
+
};
|
|
2389
|
+
this.pendingSubscriberResponses.set(shardTopic, created);
|
|
2390
|
+
// Leading edge: respond to the burst opener right away.
|
|
2391
|
+
void this.flushSubscriberResponses(shardTopic, {
|
|
2392
|
+
targets: new Set([targetHash]),
|
|
2393
|
+
topics: new Set(topics),
|
|
2394
|
+
}).catch(logErrorIfStarted);
|
|
2395
|
+
return;
|
|
2396
|
+
}
|
|
2397
|
+
entry.targets.add(targetHash);
|
|
2398
|
+
for (const t of topics) entry.topics.add(t);
|
|
2399
|
+
}
|
|
2400
|
+
|
|
2401
|
+
private async flushSubscriberResponses(
|
|
2402
|
+
shardTopic: string,
|
|
2403
|
+
entry: { targets: Set<string>; topics: Set<string> },
|
|
2404
|
+
) {
|
|
2405
|
+
if (!this.started) return;
|
|
2406
|
+
// Re-filter: our subscriptions may have changed within the window.
|
|
2407
|
+
const topics = this.getSubscriptionOverlap([...entry.topics]);
|
|
2408
|
+
if (topics.length === 0) return;
|
|
2409
|
+
const response = new Subscribe({
|
|
2410
|
+
topics,
|
|
2411
|
+
requestSubscribers: false,
|
|
2412
|
+
});
|
|
2413
|
+
const embedded = await this.createMessage(
|
|
2414
|
+
this.encodePubSubMessage(response),
|
|
2415
|
+
{
|
|
2416
|
+
mode: new AnyWhere(),
|
|
2417
|
+
priority: 1,
|
|
2418
|
+
skipRecipientValidation: true,
|
|
2419
|
+
} as any,
|
|
2420
|
+
);
|
|
2421
|
+
const payload = toUint8Array(embedded.bytes());
|
|
2422
|
+
const [firstTarget] = entry.targets;
|
|
2423
|
+
if (entry.targets.size === 1 && firstTarget) {
|
|
2424
|
+
await this.sendFanoutUnicastOrBroadcast(shardTopic, firstTarget, payload);
|
|
2425
|
+
return;
|
|
2426
|
+
}
|
|
2427
|
+
const st = this.fanoutChannels.get(shardTopic);
|
|
2428
|
+
if (!st) return;
|
|
2429
|
+
await st.channel.publishMaybe(payload);
|
|
2430
|
+
}
|
|
2431
|
+
|
|
2049
2432
|
private async sendFanoutUnicastOrBroadcast(
|
|
2050
2433
|
shardTopic: string,
|
|
2051
2434
|
targetHash: string,
|
|
@@ -2127,7 +2510,7 @@ export class TopicControlPlane
|
|
|
2127
2510
|
this.initializePeer(sender);
|
|
2128
2511
|
|
|
2129
2512
|
const existing = peers.get(senderKey);
|
|
2130
|
-
if (
|
|
2513
|
+
if (this.subscribeShouldReplace(existing, message.header.session)) {
|
|
2131
2514
|
peers.delete(senderKey);
|
|
2132
2515
|
peers.set(
|
|
2133
2516
|
senderKey,
|
|
@@ -2140,7 +2523,7 @@ export class TopicControlPlane
|
|
|
2140
2523
|
changed.push(topic);
|
|
2141
2524
|
} else {
|
|
2142
2525
|
peers.delete(senderKey);
|
|
2143
|
-
peers.set(senderKey, existing);
|
|
2526
|
+
peers.set(senderKey, existing!);
|
|
2144
2527
|
}
|
|
2145
2528
|
|
|
2146
2529
|
if (!existing) {
|
|
@@ -2236,7 +2619,7 @@ export class TopicControlPlane
|
|
|
2236
2619
|
this.initializePeer(sender);
|
|
2237
2620
|
|
|
2238
2621
|
const existing = peers.get(senderKey);
|
|
2239
|
-
if (
|
|
2622
|
+
if (this.subscribeShouldReplace(existing, message.header.session)) {
|
|
2240
2623
|
peers.delete(senderKey);
|
|
2241
2624
|
peers.set(
|
|
2242
2625
|
senderKey,
|
|
@@ -2249,7 +2632,7 @@ export class TopicControlPlane
|
|
|
2249
2632
|
changed.push(topic);
|
|
2250
2633
|
} else {
|
|
2251
2634
|
peers.delete(senderKey);
|
|
2252
|
-
peers.set(senderKey, existing);
|
|
2635
|
+
peers.set(senderKey, existing!);
|
|
2253
2636
|
}
|
|
2254
2637
|
|
|
2255
2638
|
if (!existing) {
|
|
@@ -2270,24 +2653,7 @@ export class TopicControlPlane
|
|
|
2270
2653
|
if (pubsubMessage.requestSubscribers) {
|
|
2271
2654
|
const overlap = this.getSubscriptionOverlap(pubsubMessage.topics);
|
|
2272
2655
|
if (overlap.length > 0) {
|
|
2273
|
-
|
|
2274
|
-
topics: overlap,
|
|
2275
|
-
requestSubscribers: false,
|
|
2276
|
-
});
|
|
2277
|
-
const embedded = await this.createMessage(
|
|
2278
|
-
toUint8Array(response.bytes()),
|
|
2279
|
-
{
|
|
2280
|
-
mode: new AnyWhere(),
|
|
2281
|
-
priority: 1,
|
|
2282
|
-
skipRecipientValidation: true,
|
|
2283
|
-
} as any,
|
|
2284
|
-
);
|
|
2285
|
-
const payload = toUint8Array(embedded.bytes());
|
|
2286
|
-
await this.sendFanoutUnicastOrBroadcast(
|
|
2287
|
-
shardTopic,
|
|
2288
|
-
senderKey,
|
|
2289
|
-
payload,
|
|
2290
|
-
);
|
|
2656
|
+
this.queueSubscriberResponse(shardTopic, senderKey, overlap);
|
|
2291
2657
|
}
|
|
2292
2658
|
}
|
|
2293
2659
|
return;
|
|
@@ -2406,7 +2772,7 @@ export class TopicControlPlane
|
|
|
2406
2772
|
requestSubscribers: false,
|
|
2407
2773
|
});
|
|
2408
2774
|
const embedded = await this.createMessage(
|
|
2409
|
-
|
|
2775
|
+
this.encodePubSubMessage(response),
|
|
2410
2776
|
{
|
|
2411
2777
|
mode: new AnyWhere(),
|
|
2412
2778
|
priority: 1,
|
|
@@ -2432,7 +2798,7 @@ export class TopicControlPlane
|
|
|
2432
2798
|
|
|
2433
2799
|
let pubsubMessage: PubSubMessage;
|
|
2434
2800
|
try {
|
|
2435
|
-
pubsubMessage =
|
|
2801
|
+
pubsubMessage = this.decodePubSubMessage(message.data);
|
|
2436
2802
|
} catch {
|
|
2437
2803
|
return super.onDataMessage(from, stream, message, seenBefore);
|
|
2438
2804
|
}
|