@peerbit/pubsub 5.2.11 → 5.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.d.ts +3 -0
  2. package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.d.ts.map +1 -0
  3. package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.js +90 -0
  4. package/dist/benchmark/fanout-tree-parent-upgrade-default-ready.js.map +1 -0
  5. package/dist/benchmark/fanout-tree-parent-upgrade-eval.d.ts +2 -0
  6. package/dist/benchmark/fanout-tree-parent-upgrade-eval.d.ts.map +1 -0
  7. package/dist/benchmark/fanout-tree-parent-upgrade-eval.js +712 -0
  8. package/dist/benchmark/fanout-tree-parent-upgrade-eval.js.map +1 -0
  9. package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.d.ts +2 -0
  10. package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.d.ts.map +1 -0
  11. package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.js +1591 -0
  12. package/dist/benchmark/fanout-tree-parent-upgrade-multi-eval.js.map +1 -0
  13. package/dist/benchmark/fanout-tree-parent-upgrade-prepush.d.ts +2 -0
  14. package/dist/benchmark/fanout-tree-parent-upgrade-prepush.d.ts.map +1 -0
  15. package/dist/benchmark/fanout-tree-parent-upgrade-prepush.js +297 -0
  16. package/dist/benchmark/fanout-tree-parent-upgrade-prepush.js.map +1 -0
  17. package/dist/benchmark/fanout-tree-parent-upgrade-preset.d.ts +73 -0
  18. package/dist/benchmark/fanout-tree-parent-upgrade-preset.d.ts.map +1 -0
  19. package/dist/benchmark/fanout-tree-parent-upgrade-preset.js +183 -0
  20. package/dist/benchmark/fanout-tree-parent-upgrade-preset.js.map +1 -0
  21. package/dist/benchmark/fanout-tree-sim-lib.d.ts +101 -0
  22. package/dist/benchmark/fanout-tree-sim-lib.d.ts.map +1 -1
  23. package/dist/benchmark/fanout-tree-sim-lib.js +540 -38
  24. package/dist/benchmark/fanout-tree-sim-lib.js.map +1 -1
  25. package/dist/benchmark/fanout-tree-sim.d.ts +0 -9
  26. package/dist/benchmark/fanout-tree-sim.d.ts.map +1 -1
  27. package/dist/benchmark/fanout-tree-sim.js +369 -28
  28. package/dist/benchmark/fanout-tree-sim.js.map +1 -1
  29. package/dist/benchmark/index.js +20 -0
  30. package/dist/benchmark/index.js.map +1 -1
  31. package/dist/benchmark/pubsub-topic-sim-lib.d.ts +4 -0
  32. package/dist/benchmark/pubsub-topic-sim-lib.d.ts.map +1 -1
  33. package/dist/benchmark/pubsub-topic-sim-lib.js +58 -3
  34. package/dist/benchmark/pubsub-topic-sim-lib.js.map +1 -1
  35. package/dist/benchmark/pubsub-topic-sim.js +18 -0
  36. package/dist/benchmark/pubsub-topic-sim.js.map +1 -1
  37. package/dist/src/debounced-set.d.ts +1 -1
  38. package/dist/src/debounced-set.d.ts.map +1 -1
  39. package/dist/src/debounced-set.js +5 -1
  40. package/dist/src/debounced-set.js.map +1 -1
  41. package/dist/src/fanout-tree-codec.d.ts +330 -0
  42. package/dist/src/fanout-tree-codec.d.ts.map +1 -0
  43. package/dist/src/fanout-tree-codec.js +1209 -0
  44. package/dist/src/fanout-tree-codec.js.map +1 -0
  45. package/dist/src/fanout-tree-parent-upgrade.d.ts +117 -0
  46. package/dist/src/fanout-tree-parent-upgrade.d.ts.map +1 -0
  47. package/dist/src/fanout-tree-parent-upgrade.js +135 -0
  48. package/dist/src/fanout-tree-parent-upgrade.js.map +1 -0
  49. package/dist/src/fanout-tree.d.ts +245 -73
  50. package/dist/src/fanout-tree.d.ts.map +1 -1
  51. package/dist/src/fanout-tree.js +2290 -1057
  52. package/dist/src/fanout-tree.js.map +1 -1
  53. package/dist/src/index.d.ts +58 -0
  54. package/dist/src/index.d.ts.map +1 -1
  55. package/dist/src/index.js +316 -45
  56. package/dist/src/index.js.map +1 -1
  57. package/dist/src/topic-root-control-plane.d.ts +13 -0
  58. package/dist/src/topic-root-control-plane.d.ts.map +1 -1
  59. package/dist/src/topic-root-control-plane.js +45 -0
  60. package/dist/src/topic-root-control-plane.js.map +1 -1
  61. package/package.json +9 -12
  62. package/src/debounced-set.ts +5 -0
  63. package/src/fanout-tree-codec.ts +1580 -0
  64. package/src/fanout-tree-parent-upgrade.ts +353 -0
  65. package/src/fanout-tree.ts +4558 -2230
  66. package/src/index.ts +425 -59
  67. package/src/topic-root-control-plane.ts +48 -0
package/src/index.ts CHANGED
@@ -28,6 +28,7 @@ import {
28
28
  type DirectStreamComponents,
29
29
  type DirectStreamOptions,
30
30
  type PeerStreams,
31
+ type RustTopicControl,
31
32
  } from "@peerbit/stream";
32
33
  import {
33
34
  AcknowledgeAnyWhere,
@@ -64,12 +65,30 @@ import type {
64
65
  import { TopicRootControlPlane } from "./topic-root-control-plane.js";
65
66
 
66
67
  export * from "./fanout-tree.js";
68
+ // The complete /peerbit/fanout-tree/0.5.0 wire codec and the
69
+ // parent-upgrade decision module (the TS references the native rust-core
70
+ // implementations are parity-tested against).
71
+ export * as fanoutWire from "./fanout-tree-codec.js";
72
+ export * as fanoutParentUpgrade from "./fanout-tree-parent-upgrade.js";
67
73
  export * from "./fanout-channel.js";
68
74
  export * from "./topic-root-control-plane.js";
69
75
  export { toUint8Array } from "./bytes.js";
70
76
 
71
77
  export const logger = loggerFn("peerbit:transport:topic-control-plane");
72
78
  const warn = logger.newScope("warn");
79
+
80
+ const utf8Encoder = new TextEncoder();
81
+ /**
82
+ * Offset of the `data` bytes inside a borsh-serialized `PubSubData`
83
+ * (variant byte + topics vec + strict bool + data length prefix).
84
+ */
85
+ const preEncodedPubSubDataOffset = (topics: string[]): number => {
86
+ let offset = 1 + 4; // variant + topics vec length
87
+ for (const topic of topics) {
88
+ offset += 4 + utf8Encoder.encode(topic).byteLength;
89
+ }
90
+ return offset + 1 + 4; // strict + data length prefix
91
+ };
73
92
  const logError = (e?: { message: string }) => {
74
93
  logger.error(e?.message);
75
94
  };
@@ -119,6 +138,7 @@ const AUTO_TOPIC_ROOT_CANDIDATES_MAX = 64;
119
138
  // Topic-root queries may need to wait for the responder to finish opening an
120
139
  // outbound stream back to the requester after an inbound-only dial.
121
140
  const DEFAULT_TOPIC_ROOT_QUERY_TIMEOUT_MS = 12_000;
141
+ const DIRECT_SHARD_ROOT_CONFIRM_TIMEOUT_MS = 2_000;
122
142
 
123
143
  const DEFAULT_PUBSUB_FANOUT_CHANNEL_OPTIONS: Omit<
124
144
  FanoutTreeChannelOptions,
@@ -238,6 +258,21 @@ export class TopicControlPlane
238
258
  public subscriptions: Map<string, { counter: number }>;
239
259
  // Local topics requested via debounced subscribe, not yet applied in `subscriptions`.
240
260
  private pendingSubscriptions: Set<string>;
261
+ /**
262
+ * Per-shard batching of Subscribe{requestSubscribers} responses. When many
263
+ * peers announce subscriptions in a burst (group subscribe), answering each
264
+ * announcer with a routed unicast is quadratic in subscriber count and
265
+ * melts the fanout route-resolution machinery (#983); a short window lets
266
+ * one broadcast answer all concurrent announcers instead.
267
+ */
268
+ private pendingSubscriberResponses: Map<
269
+ string,
270
+ {
271
+ targets: Set<string>;
272
+ topics: Set<string>;
273
+ timer: ReturnType<typeof setTimeout>;
274
+ }
275
+ > = new Map();
241
276
  public lastSubscriptionMessages: Map<
242
277
  string,
243
278
  Map<string, { session: bigint; timestamp: bigint }>
@@ -246,6 +281,13 @@ export class TopicControlPlane
246
281
  public readonly topicRootControlPlane: TopicRootControlPlane;
247
282
  public readonly subscriberCacheMaxEntries: number;
248
283
  public readonly fanout: FanoutTree;
284
+ // Native topic-control components of the rust-core DirectStream engine
285
+ // (`@peerbit/network-rust`). When set, the PubSubMessage codec, the topic
286
+ // hashing behind shard mapping and root selection, the root-directory
287
+ // state and the subscribe-state convergence rules run natively; the
288
+ // observable subscription maps and all events stay unchanged. Unset (the
289
+ // default) leaves every code path as-is.
290
+ private readonly nativeTopicControl?: RustTopicControl;
249
291
 
250
292
  private debounceSubscribeAggregator: DebouncedAccumulatorCounterMap;
251
293
  private debounceUnsubscribeAggregator: DebouncedAccumulatorCounterMap;
@@ -253,10 +295,7 @@ export class TopicControlPlane
253
295
  private readonly shardCount: number;
254
296
  private readonly shardTopicPrefix: string;
255
297
  private readonly hostShards: boolean;
256
- private readonly shardRootCache = new Map<
257
- string,
258
- { root: string; authoritative: boolean }
259
- >();
298
+ private readonly shardRootCache = new Map<string, { root: string; authoritative: boolean }>();
260
299
  private readonly shardTopicCache = new Map<string, string>();
261
300
  private readonly shardRefCounts = new Map<string, number>();
262
301
  private readonly pinnedShards = new Set<string>();
@@ -325,6 +364,12 @@ export class TopicControlPlane
325
364
 
326
365
  this.topicRootControlPlane =
327
366
  props?.topicRootControlPlane || new TopicRootControlPlane();
367
+ this.nativeTopicControl = this.rustCore?.topicControl;
368
+ if (this.nativeTopicControl) {
369
+ this.topicRootControlPlane.adoptNativeDirectoryState(
370
+ this.nativeTopicControl.createRootDirectoryState(),
371
+ );
372
+ }
328
373
  this.dispatchEventOnSelfPublish =
329
374
  props?.dispatchEventOnSelfPublish || false;
330
375
 
@@ -393,12 +438,18 @@ export class TopicControlPlane
393
438
  this.debounceSubscribeAggregator = debouncedAccumulatorSetCounter(
394
439
  (set) => this._subscribe([...set.values()]),
395
440
  props?.subscriptionDebounceDelay ?? 50,
441
+ (error) =>
442
+ logger.error(`Debounced subscribe failed: ${error?.message ?? error}`),
396
443
  );
397
444
  // NOTE: Unsubscribe should update local state immediately and batch only the
398
445
  // best-effort network announcements to avoid teardown stalls (program close).
399
446
  this.debounceUnsubscribeAggregator = debouncedAccumulatorSetCounter(
400
447
  (set) => this._announceUnsubscribe([...set.values()]),
401
448
  props?.subscriptionDebounceDelay ?? 50,
449
+ (error) =>
450
+ logger.error(
451
+ `Debounced unsubscribe announce failed: ${error?.message ?? error}`,
452
+ ),
402
453
  );
403
454
  }
404
455
 
@@ -497,6 +548,10 @@ export class TopicControlPlane
497
548
 
498
549
  this.subscriptions.clear();
499
550
  this.pendingSubscriptions.clear();
551
+ for (const entry of this.pendingSubscriberResponses.values()) {
552
+ clearTimeout(entry.timer);
553
+ }
554
+ this.pendingSubscriberResponses.clear();
500
555
  this.topics.clear();
501
556
  this.peerToTopic.clear();
502
557
  this.lastSubscriptionMessages.clear();
@@ -611,6 +666,12 @@ export class TopicControlPlane
611
666
  }
612
667
 
613
668
  private normalizeAutoTopicRootCandidates(candidates: string[]): string[] {
669
+ if (this.nativeTopicControl) {
670
+ return this.nativeTopicControl.normalizeAutoCandidates(
671
+ candidates,
672
+ this.publicKeyHash,
673
+ );
674
+ }
614
675
  const unique = new Set<string>();
615
676
  for (const c of candidates) {
616
677
  if (!c) continue;
@@ -680,7 +741,7 @@ export class TopicControlPlane
680
741
  if (candidates.length === 0) return;
681
742
 
682
743
  const msg = new TopicRootCandidates({ candidates });
683
- const embedded = await this.createMessage(toUint8Array(msg.bytes()), {
744
+ const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
684
745
  mode: new AnyWhere(),
685
746
  priority: 1,
686
747
  skipRecipientValidation: true,
@@ -776,7 +837,7 @@ export class TopicControlPlane
776
837
  topics: userTopics,
777
838
  requestSubscribers: true,
778
839
  });
779
- const embedded = await this.createMessage(toUint8Array(msg.bytes()), {
840
+ const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
780
841
  mode: new AnyWhere(),
781
842
  priority: 1,
782
843
  skipRecipientValidation: true,
@@ -925,12 +986,118 @@ export class TopicControlPlane
925
986
  const t = topic.toString();
926
987
  const cached = this.shardTopicCache.get(t);
927
988
  if (cached) return cached;
928
- const index = topicHash32(t) % this.shardCount;
929
- const shardTopic = `${this.shardTopicPrefix}${index}`;
989
+ const shardTopic = this.nativeTopicControl
990
+ ? this.nativeTopicControl.shardTopic(
991
+ t,
992
+ this.shardCount,
993
+ this.shardTopicPrefix,
994
+ )
995
+ : `${this.shardTopicPrefix}${topicHash32(t) % this.shardCount}`;
930
996
  this.shardTopicCache.set(t, shardTopic);
931
997
  return shardTopic;
932
998
  }
933
999
 
1000
+ /**
1001
+ * Serialize a control-plane message (borsh `PubSubMessage`); rust-core
1002
+ * mode encodes natively (byte-identical, covered by the parity suite).
1003
+ */
1004
+ private encodePubSubMessage(message: PubSubMessage): Uint8Array {
1005
+ const native = this.nativeTopicControl;
1006
+ if (native) {
1007
+ if (message instanceof PubSubData) {
1008
+ return native.encodePubSubData(
1009
+ message.topics,
1010
+ message.strict,
1011
+ message.data,
1012
+ );
1013
+ }
1014
+ if (message instanceof Subscribe) {
1015
+ return native.encodeSubscribe(
1016
+ message.topics,
1017
+ message.requestSubscribers,
1018
+ );
1019
+ }
1020
+ if (message instanceof Unsubscribe) {
1021
+ return native.encodeUnsubscribe(message.topics);
1022
+ }
1023
+ if (message instanceof GetSubscribers) {
1024
+ return native.encodeGetSubscribers(message.topics);
1025
+ }
1026
+ if (message instanceof TopicRootCandidates) {
1027
+ return native.encodeTopicRootCandidates(message.candidates);
1028
+ }
1029
+ if (message instanceof PeerUnavailable) {
1030
+ return native.encodePeerUnavailable(
1031
+ message.publicKeyHash,
1032
+ message.session,
1033
+ message.timestamp,
1034
+ message.topics,
1035
+ );
1036
+ }
1037
+ if (message instanceof TopicRootQuery) {
1038
+ return native.encodeTopicRootQuery(message.requestId, message.topic);
1039
+ }
1040
+ if (message instanceof TopicRootQueryResponse) {
1041
+ return native.encodeTopicRootQueryResponse(
1042
+ message.requestId,
1043
+ message.topic,
1044
+ message.root,
1045
+ );
1046
+ }
1047
+ }
1048
+ return toUint8Array(message.bytes());
1049
+ }
1050
+
1051
+ /**
1052
+ * Parse a control-plane payload; rust-core mode decodes natively into the
1053
+ * same message classes (decode failures throw either way, so callers keep
1054
+ * their fallback behavior).
1055
+ */
1056
+ private decodePubSubMessage(bytes: Uint8Array): PubSubMessage {
1057
+ const native = this.nativeTopicControl;
1058
+ if (native) {
1059
+ const decoded = native.decodePubSubMessage(bytes);
1060
+ switch (decoded.type) {
1061
+ case "data":
1062
+ return new PubSubData({
1063
+ topics: decoded.topics,
1064
+ data: decoded.data,
1065
+ strict: decoded.strict,
1066
+ });
1067
+ case "subscribe":
1068
+ return new Subscribe({
1069
+ topics: decoded.topics,
1070
+ requestSubscribers: decoded.requestSubscribers,
1071
+ });
1072
+ case "unsubscribe":
1073
+ return new Unsubscribe({ topics: decoded.topics });
1074
+ case "get-subscribers":
1075
+ return new GetSubscribers({ topics: decoded.topics });
1076
+ case "topic-root-candidates":
1077
+ return new TopicRootCandidates({ candidates: decoded.candidates });
1078
+ case "peer-unavailable":
1079
+ return new PeerUnavailable({
1080
+ publicKeyHash: decoded.publicKeyHash,
1081
+ session: decoded.session,
1082
+ timestamp: decoded.timestamp,
1083
+ topics: decoded.topics,
1084
+ });
1085
+ case "topic-root-query":
1086
+ return new TopicRootQuery({
1087
+ requestId: decoded.requestId,
1088
+ topic: decoded.topic,
1089
+ });
1090
+ case "topic-root-query-response":
1091
+ return new TopicRootQueryResponse({
1092
+ requestId: decoded.requestId,
1093
+ topic: decoded.topic,
1094
+ root: decoded.root,
1095
+ });
1096
+ }
1097
+ }
1098
+ return PubSubMessage.from(bytes);
1099
+ }
1100
+
934
1101
  private async resolveTopicRootState(
935
1102
  topic: string,
936
1103
  ): Promise<{ root?: string; authoritative: boolean }> {
@@ -1024,7 +1191,7 @@ export class TopicControlPlane
1024
1191
  peer: PeerStreams,
1025
1192
  pubsubMessage: PubSubMessage,
1026
1193
  ) {
1027
- const embedded = await this.createMessage(toUint8Array(pubsubMessage.bytes()), {
1194
+ const embedded = await this.createMessage(this.encodePubSubMessage(pubsubMessage), {
1028
1195
  mode: new SilentDelivery({
1029
1196
  to: [peer.publicKey.hashcode()],
1030
1197
  redundancy: 1,
@@ -1051,6 +1218,7 @@ export class TopicControlPlane
1051
1218
  private async queryTopicRootFromPeer(
1052
1219
  peer: PeerStreams,
1053
1220
  topic: string,
1221
+ timeoutMs = DEFAULT_TOPIC_ROOT_QUERY_TIMEOUT_MS,
1054
1222
  ): Promise<string | undefined> {
1055
1223
  if (!this.started || this.stopping) return undefined;
1056
1224
 
@@ -1059,7 +1227,7 @@ export class TopicControlPlane
1059
1227
  const timer = setTimeout(() => {
1060
1228
  this.pendingTopicRootQueries.delete(requestId);
1061
1229
  resolve(undefined);
1062
- }, DEFAULT_TOPIC_ROOT_QUERY_TIMEOUT_MS);
1230
+ }, Math.max(1, Math.floor(timeoutMs)));
1063
1231
  timer.unref?.();
1064
1232
  this.pendingTopicRootQueries.set(requestId, { topic, resolve, timer });
1065
1233
  });
@@ -1081,6 +1249,33 @@ export class TopicControlPlane
1081
1249
  return responsePromise;
1082
1250
  }
1083
1251
 
1252
+ private async confirmDirectShardRoot(
1253
+ shardTopic: string,
1254
+ root: string,
1255
+ signal?: AbortSignal,
1256
+ ): Promise<string> {
1257
+ if (root === this.publicKeyHash) return root;
1258
+ const rootPeer = this.peers.get(root);
1259
+ if (!rootPeer) return root;
1260
+
1261
+ const confirmation = await withAbort(
1262
+ this.queryTopicRootFromPeer(
1263
+ rootPeer,
1264
+ shardTopic,
1265
+ DIRECT_SHARD_ROOT_CONFIRM_TIMEOUT_MS,
1266
+ ),
1267
+ signal,
1268
+ );
1269
+ if (!confirmation) return root;
1270
+ if (confirmation !== root) {
1271
+ this.shardRootCache.set(shardTopic, {
1272
+ root: confirmation,
1273
+ authoritative: true,
1274
+ });
1275
+ }
1276
+ return confirmation;
1277
+ }
1278
+
1084
1279
  private async resolveQueryableTopicRoot(
1085
1280
  topic: string,
1086
1281
  ): Promise<string | undefined> {
@@ -1137,6 +1332,7 @@ export class TopicControlPlane
1137
1332
  signal?: AbortSignal;
1138
1333
  },
1139
1334
  ): Promise<void> {
1335
+ if (!this.started) throw new NotStartedError();
1140
1336
  const t = shardTopic.toString();
1141
1337
  const pin = options?.pin === true;
1142
1338
  const wantEphemeral = options?.ephemeral === true;
@@ -1166,6 +1362,7 @@ export class TopicControlPlane
1166
1362
  }
1167
1363
 
1168
1364
  root = root ?? (await this.resolveShardRoot(t));
1365
+ root = await this.confirmDirectShardRoot(t, root, options?.signal);
1169
1366
  const channel = new FanoutChannel(this.fanout, { topic: t, root });
1170
1367
 
1171
1368
  const onPayload = (payload: Uint8Array) => {
@@ -1186,7 +1383,7 @@ export class TopicControlPlane
1186
1383
 
1187
1384
  let pubsubMessage: PubSubMessage;
1188
1385
  try {
1189
- pubsubMessage = PubSubMessage.from(dm.data);
1386
+ pubsubMessage = this.decodePubSubMessage(dm.data);
1190
1387
  } catch {
1191
1388
  return;
1192
1389
  }
@@ -1232,6 +1429,9 @@ export class TopicControlPlane
1232
1429
  this.seenCache.add(msgId, seen ? seen + 1 : 1);
1233
1430
  if (seen) return;
1234
1431
 
1432
+ // fanout-delivered frames bypass the inbound stream decode, so
1433
+ // the nested envelope is verified natively here when possible
1434
+ this.seedNativeWireVerification(dm, payload);
1235
1435
  if ((await this.verifyAndProcess(dm)) === false) {
1236
1436
  return;
1237
1437
  }
@@ -1295,9 +1495,9 @@ export class TopicControlPlane
1295
1495
  // ignore
1296
1496
  }
1297
1497
  try {
1298
- channel.close();
1498
+ await channel.leave({ notifyParent: false, kickChildren: false });
1299
1499
  } catch {
1300
- // ignore
1500
+ channel.close();
1301
1501
  }
1302
1502
  throw error;
1303
1503
  }
@@ -1419,7 +1619,7 @@ export class TopicControlPlane
1419
1619
  topics: userTopics,
1420
1620
  requestSubscribers: true,
1421
1621
  });
1422
- const embedded = await this.createMessage(toUint8Array(msg.bytes()), {
1622
+ const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
1423
1623
  mode: new AnyWhere(),
1424
1624
  priority: 1,
1425
1625
  skipRecipientValidation: true,
@@ -1498,7 +1698,7 @@ export class TopicControlPlane
1498
1698
  // Announce first.
1499
1699
  try {
1500
1700
  const msg = new Unsubscribe({ topics: userTopics });
1501
- const embedded = await this.createMessage(toUint8Array(msg.bytes()), {
1701
+ const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
1502
1702
  mode: new AnyWhere(),
1503
1703
  priority: 1,
1504
1704
  skipRecipientValidation: true,
@@ -1571,7 +1771,7 @@ export class TopicControlPlane
1571
1771
  timestamp: batch.timestamp,
1572
1772
  topics: batch.topics,
1573
1773
  });
1574
- const embedded = await this.createMessage(toUint8Array(msg.bytes()), {
1774
+ const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
1575
1775
  mode: new AnyWhere(),
1576
1776
  priority: 1,
1577
1777
  skipRecipientValidation: true,
@@ -1601,7 +1801,7 @@ export class TopicControlPlane
1601
1801
  timestamp: 0n,
1602
1802
  topics: [],
1603
1803
  });
1604
- const embedded = await this.createMessage(toUint8Array(msg.bytes()), {
1804
+ const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
1605
1805
  mode: new AnyWhere(),
1606
1806
  priority: 1,
1607
1807
  skipRecipientValidation: true,
@@ -1702,7 +1902,7 @@ export class TopicControlPlane
1702
1902
  const persistent = (this.shardRefCounts.get(shardTopic) ?? 0) > 0;
1703
1903
  await this.ensureFanoutChannel(shardTopic, { ephemeral: !persistent });
1704
1904
 
1705
- const embedded = await this.createMessage(toUint8Array(msg.bytes()), {
1905
+ const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
1706
1906
  mode: new AnyWhere(),
1707
1907
  priority: 1,
1708
1908
  skipRecipientValidation: true,
@@ -1754,10 +1954,13 @@ export class TopicControlPlane
1754
1954
  const msg = data
1755
1955
  ? new PubSubData({ topics: topicsAll, data, strict: true })
1756
1956
  : undefined;
1757
- const message = await this.createMessage(msg?.bytes(), {
1758
- ...options,
1759
- skipRecipientValidation: this.dispatchEventOnSelfPublish,
1760
- });
1957
+ const message = await this.createMessage(
1958
+ msg ? this.encodePubSubMessage(msg) : undefined,
1959
+ {
1960
+ ...options,
1961
+ skipRecipientValidation: this.dispatchEventOnSelfPublish,
1962
+ },
1963
+ );
1761
1964
 
1762
1965
  if (msg) {
1763
1966
  this.dispatchEvent(
@@ -1800,7 +2003,7 @@ export class TopicControlPlane
1800
2003
  }
1801
2004
 
1802
2005
  const msg = new PubSubData({ topics: topicsAll, data, strict: false });
1803
- const embedded = await this.createMessage(toUint8Array(msg.bytes()), {
2006
+ const embedded = await this.createMessage(this.encodePubSubMessage(msg), {
1804
2007
  mode: new AnyWhere(),
1805
2008
  priority: options?.priority,
1806
2009
  responsePriority: options?.responsePriority,
@@ -1862,6 +2065,76 @@ export class TopicControlPlane
1862
2065
  return embedded.id;
1863
2066
  }
1864
2067
 
2068
+ /**
2069
+ * Publish a pre-encoded `PubSubData` payload to explicit recipients.
2070
+ *
2071
+ * `payload` must be the exact borsh serialization of
2072
+ * `new PubSubData({ topics, data, strict: true })` — callers that encode
2073
+ * the payload elsewhere (e.g. the shared-log fused send path serializing
2074
+ * inside wasm) hand the finished bytes here so the wrapping `PubSubData`
2075
+ * copy of the regular `publish` path is skipped. Everything else matches
2076
+ * the explicit-recipient branch of {@link publish}: the payload becomes a
2077
+ * signed `DataMessage` delivered over DirectStream.
2078
+ */
2079
+ async publishPreEncodedData(
2080
+ payload: Uint8Array,
2081
+ properties: {
2082
+ topics: string[];
2083
+ },
2084
+ options: {
2085
+ mode: SilentDelivery | AcknowledgeDelivery;
2086
+ } & { client?: string } & PriorityOptions &
2087
+ ResponsePriorityOptions &
2088
+ ExpiresAtOptions &
2089
+ IdOptions &
2090
+ WithExtraSigners & { signal?: AbortSignal },
2091
+ ): Promise<Uint8Array | undefined> {
2092
+ if (!this.started) throw new NotStartedError();
2093
+ if (!options?.mode || !deliveryModeHasReceiver(options.mode)) {
2094
+ throw new Error(
2095
+ "publishPreEncodedData requires a delivery mode with explicit recipients",
2096
+ );
2097
+ }
2098
+
2099
+ const message = await this.createMessage(payload, {
2100
+ ...options,
2101
+ skipRecipientValidation: this.dispatchEventOnSelfPublish,
2102
+ });
2103
+
2104
+ this.dispatchEvent(
2105
+ new CustomEvent("publish", {
2106
+ detail: new PublishEvent({
2107
+ client: options?.client,
2108
+ data: new PubSubData({
2109
+ topics: properties.topics,
2110
+ data: payload.subarray(
2111
+ preEncodedPubSubDataOffset(properties.topics),
2112
+ ),
2113
+ strict: true,
2114
+ }),
2115
+ message,
2116
+ }),
2117
+ }),
2118
+ );
2119
+
2120
+ const silentDelivery = options.mode instanceof SilentDelivery;
2121
+ try {
2122
+ await this.publishMessage(
2123
+ this.publicKey,
2124
+ message,
2125
+ undefined,
2126
+ undefined,
2127
+ options?.signal,
2128
+ );
2129
+ } catch (error) {
2130
+ if (error instanceof DeliveryError && silentDelivery !== false) {
2131
+ return message.id;
2132
+ }
2133
+ throw error;
2134
+ }
2135
+ return message.id;
2136
+ }
2137
+
1865
2138
  public onPeerSession(key: PublicSignKey, _session: number): void {
1866
2139
  if (!Number.isFinite(_session) || _session < 0) {
1867
2140
  return;
@@ -1999,23 +2272,44 @@ export class TopicControlPlane
1999
2272
  timestamp: bigint,
2000
2273
  relevantTopics: string[],
2001
2274
  ) {
2002
- for (const topic of relevantTopics) {
2003
- const last = this.lastSubscriptionMessages
2004
- .get(subscriberKey)
2005
- ?.get(topic);
2006
- if (!last) {
2007
- continue;
2008
- }
2009
- if (last.session > session) {
2010
- return false;
2275
+ if (this.nativeTopicControl) {
2276
+ const lasts: bigint[] = [];
2277
+ for (const topic of relevantTopics) {
2278
+ const last = this.lastSubscriptionMessages
2279
+ .get(subscriberKey)
2280
+ ?.get(topic);
2281
+ if (last) {
2282
+ lasts.push(last.session, last.timestamp);
2283
+ }
2011
2284
  }
2012
2285
  if (
2013
- timestamp !== 0n &&
2014
- last.session === session &&
2015
- last.timestamp > timestamp
2286
+ !this.nativeTopicControl.subscriptionIsLatest(
2287
+ BigUint64Array.from(lasts),
2288
+ session,
2289
+ timestamp,
2290
+ )
2016
2291
  ) {
2017
2292
  return false;
2018
2293
  }
2294
+ } else {
2295
+ for (const topic of relevantTopics) {
2296
+ const last = this.lastSubscriptionMessages
2297
+ .get(subscriberKey)
2298
+ ?.get(topic);
2299
+ if (!last) {
2300
+ continue;
2301
+ }
2302
+ if (last.session > session) {
2303
+ return false;
2304
+ }
2305
+ if (
2306
+ timestamp !== 0n &&
2307
+ last.session === session &&
2308
+ last.timestamp > timestamp
2309
+ ) {
2310
+ return false;
2311
+ }
2312
+ }
2019
2313
  }
2020
2314
 
2021
2315
  for (const topic of relevantTopics) {
@@ -2032,6 +2326,23 @@ export class TopicControlPlane
2032
2326
  return true;
2033
2327
  }
2034
2328
 
2329
+ /**
2330
+ * `Subscribe` replaces tracked subscription data for new subscribers or
2331
+ * strictly newer sessions; equal/older sessions only refresh cache order.
2332
+ */
2333
+ private subscribeShouldReplace(
2334
+ existing: SubscriptionData | undefined,
2335
+ session: bigint,
2336
+ ): boolean {
2337
+ if (this.nativeTopicControl) {
2338
+ return this.nativeTopicControl.subscribeShouldReplace(
2339
+ existing?.session,
2340
+ session,
2341
+ );
2342
+ }
2343
+ return !existing || existing.session < session;
2344
+ }
2345
+
2035
2346
  private subscriptionMessageIsLatest(
2036
2347
  message: DataMessage,
2037
2348
  _pubsubMessage: Subscribe | Unsubscribe,
@@ -2046,6 +2357,78 @@ export class TopicControlPlane
2046
2357
  );
2047
2358
  }
2048
2359
 
2360
+ /**
2361
+ * Answer Subscribe{requestSubscribers} announces without letting bursts go
2362
+ * quadratic (#983). Leading edge: the first announcer is answered
2363
+ * immediately with a targeted unicast — a lone joiner sees no added
2364
+ * latency. Announcers arriving within the trailing window that this opens
2365
+ * are batched and answered with ONE broadcast, instead of S(S-1) routed
2366
+ * unicasts whose cold route resolutions flood the shard tree.
2367
+ */
2368
+ private queueSubscriberResponse(
2369
+ shardTopic: string,
2370
+ targetHash: string,
2371
+ topics: string[],
2372
+ ) {
2373
+ let entry = this.pendingSubscriberResponses.get(shardTopic);
2374
+ if (!entry) {
2375
+ const created = {
2376
+ targets: new Set<string>(),
2377
+ topics: new Set<string>(),
2378
+ timer: setTimeout(
2379
+ () => {
2380
+ this.pendingSubscriberResponses.delete(shardTopic);
2381
+ if (created.targets.size === 0) return;
2382
+ void this.flushSubscriberResponses(shardTopic, created).catch(
2383
+ logErrorIfStarted,
2384
+ );
2385
+ },
2386
+ 150 + Math.floor(Math.random() * 150),
2387
+ ),
2388
+ };
2389
+ this.pendingSubscriberResponses.set(shardTopic, created);
2390
+ // Leading edge: respond to the burst opener right away.
2391
+ void this.flushSubscriberResponses(shardTopic, {
2392
+ targets: new Set([targetHash]),
2393
+ topics: new Set(topics),
2394
+ }).catch(logErrorIfStarted);
2395
+ return;
2396
+ }
2397
+ entry.targets.add(targetHash);
2398
+ for (const t of topics) entry.topics.add(t);
2399
+ }
2400
+
2401
+ private async flushSubscriberResponses(
2402
+ shardTopic: string,
2403
+ entry: { targets: Set<string>; topics: Set<string> },
2404
+ ) {
2405
+ if (!this.started) return;
2406
+ // Re-filter: our subscriptions may have changed within the window.
2407
+ const topics = this.getSubscriptionOverlap([...entry.topics]);
2408
+ if (topics.length === 0) return;
2409
+ const response = new Subscribe({
2410
+ topics,
2411
+ requestSubscribers: false,
2412
+ });
2413
+ const embedded = await this.createMessage(
2414
+ this.encodePubSubMessage(response),
2415
+ {
2416
+ mode: new AnyWhere(),
2417
+ priority: 1,
2418
+ skipRecipientValidation: true,
2419
+ } as any,
2420
+ );
2421
+ const payload = toUint8Array(embedded.bytes());
2422
+ const [firstTarget] = entry.targets;
2423
+ if (entry.targets.size === 1 && firstTarget) {
2424
+ await this.sendFanoutUnicastOrBroadcast(shardTopic, firstTarget, payload);
2425
+ return;
2426
+ }
2427
+ const st = this.fanoutChannels.get(shardTopic);
2428
+ if (!st) return;
2429
+ await st.channel.publishMaybe(payload);
2430
+ }
2431
+
2049
2432
  private async sendFanoutUnicastOrBroadcast(
2050
2433
  shardTopic: string,
2051
2434
  targetHash: string,
@@ -2127,7 +2510,7 @@ export class TopicControlPlane
2127
2510
  this.initializePeer(sender);
2128
2511
 
2129
2512
  const existing = peers.get(senderKey);
2130
- if (!existing || existing.session < message.header.session) {
2513
+ if (this.subscribeShouldReplace(existing, message.header.session)) {
2131
2514
  peers.delete(senderKey);
2132
2515
  peers.set(
2133
2516
  senderKey,
@@ -2140,7 +2523,7 @@ export class TopicControlPlane
2140
2523
  changed.push(topic);
2141
2524
  } else {
2142
2525
  peers.delete(senderKey);
2143
- peers.set(senderKey, existing);
2526
+ peers.set(senderKey, existing!);
2144
2527
  }
2145
2528
 
2146
2529
  if (!existing) {
@@ -2236,7 +2619,7 @@ export class TopicControlPlane
2236
2619
  this.initializePeer(sender);
2237
2620
 
2238
2621
  const existing = peers.get(senderKey);
2239
- if (!existing || existing.session < message.header.session) {
2622
+ if (this.subscribeShouldReplace(existing, message.header.session)) {
2240
2623
  peers.delete(senderKey);
2241
2624
  peers.set(
2242
2625
  senderKey,
@@ -2249,7 +2632,7 @@ export class TopicControlPlane
2249
2632
  changed.push(topic);
2250
2633
  } else {
2251
2634
  peers.delete(senderKey);
2252
- peers.set(senderKey, existing);
2635
+ peers.set(senderKey, existing!);
2253
2636
  }
2254
2637
 
2255
2638
  if (!existing) {
@@ -2270,24 +2653,7 @@ export class TopicControlPlane
2270
2653
  if (pubsubMessage.requestSubscribers) {
2271
2654
  const overlap = this.getSubscriptionOverlap(pubsubMessage.topics);
2272
2655
  if (overlap.length > 0) {
2273
- const response = new Subscribe({
2274
- topics: overlap,
2275
- requestSubscribers: false,
2276
- });
2277
- const embedded = await this.createMessage(
2278
- toUint8Array(response.bytes()),
2279
- {
2280
- mode: new AnyWhere(),
2281
- priority: 1,
2282
- skipRecipientValidation: true,
2283
- } as any,
2284
- );
2285
- const payload = toUint8Array(embedded.bytes());
2286
- await this.sendFanoutUnicastOrBroadcast(
2287
- shardTopic,
2288
- senderKey,
2289
- payload,
2290
- );
2656
+ this.queueSubscriberResponse(shardTopic, senderKey, overlap);
2291
2657
  }
2292
2658
  }
2293
2659
  return;
@@ -2406,7 +2772,7 @@ export class TopicControlPlane
2406
2772
  requestSubscribers: false,
2407
2773
  });
2408
2774
  const embedded = await this.createMessage(
2409
- toUint8Array(response.bytes()),
2775
+ this.encodePubSubMessage(response),
2410
2776
  {
2411
2777
  mode: new AnyWhere(),
2412
2778
  priority: 1,
@@ -2432,7 +2798,7 @@ export class TopicControlPlane
2432
2798
 
2433
2799
  let pubsubMessage: PubSubMessage;
2434
2800
  try {
2435
- pubsubMessage = PubSubMessage.from(message.data);
2801
+ pubsubMessage = this.decodePubSubMessage(message.data);
2436
2802
  } catch {
2437
2803
  return super.onDataMessage(from, stream, message, seenBefore);
2438
2804
  }