@peerbit/shared-log 13.1.18 → 13.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/dist/benchmark/index.js +91 -29
  2. package/dist/benchmark/index.js.map +1 -1
  3. package/dist/benchmark/native-graph.js +120 -10
  4. package/dist/benchmark/native-graph.js.map +1 -1
  5. package/dist/benchmark/network-preset-e2e.d.ts +2 -0
  6. package/dist/benchmark/network-preset-e2e.d.ts.map +1 -0
  7. package/dist/benchmark/network-preset-e2e.js +497 -0
  8. package/dist/benchmark/network-preset-e2e.js.map +1 -0
  9. package/dist/benchmark/receive-prune.d.ts +2 -0
  10. package/dist/benchmark/receive-prune.d.ts.map +1 -0
  11. package/dist/benchmark/receive-prune.js +816 -0
  12. package/dist/benchmark/receive-prune.js.map +1 -0
  13. package/dist/benchmark/sync-catchup.d.ts +1 -2
  14. package/dist/benchmark/sync-catchup.d.ts.map +1 -1
  15. package/dist/benchmark/sync-catchup.js +175 -7
  16. package/dist/benchmark/sync-catchup.js.map +1 -1
  17. package/dist/src/checked-prune.d.ts +3 -0
  18. package/dist/src/checked-prune.d.ts.map +1 -1
  19. package/dist/src/checked-prune.js +24 -0
  20. package/dist/src/checked-prune.js.map +1 -1
  21. package/dist/src/exchange-heads.d.ts +238 -1
  22. package/dist/src/exchange-heads.d.ts.map +1 -1
  23. package/dist/src/exchange-heads.js +1512 -36
  24. package/dist/src/exchange-heads.js.map +1 -1
  25. package/dist/src/index.d.ts +276 -5
  26. package/dist/src/index.d.ts.map +1 -1
  27. package/dist/src/index.js +7358 -666
  28. package/dist/src/index.js.map +1 -1
  29. package/dist/src/pid.d.ts.map +1 -1
  30. package/dist/src/pid.js +25 -9
  31. package/dist/src/pid.js.map +1 -1
  32. package/dist/src/ranges.d.ts +11 -2
  33. package/dist/src/ranges.d.ts.map +1 -1
  34. package/dist/src/ranges.js +74 -30
  35. package/dist/src/ranges.js.map +1 -1
  36. package/dist/src/sync/index.d.ts +77 -2
  37. package/dist/src/sync/index.d.ts.map +1 -1
  38. package/dist/src/sync/rateless-iblt.d.ts +11 -5
  39. package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
  40. package/dist/src/sync/rateless-iblt.js +123 -60
  41. package/dist/src/sync/rateless-iblt.js.map +1 -1
  42. package/dist/src/sync/simple.d.ts +56 -4
  43. package/dist/src/sync/simple.d.ts.map +1 -1
  44. package/dist/src/sync/simple.js +495 -63
  45. package/dist/src/sync/simple.js.map +1 -1
  46. package/dist/src/utils.d.ts +2 -1
  47. package/dist/src/utils.d.ts.map +1 -1
  48. package/dist/src/utils.js +49 -2
  49. package/dist/src/utils.js.map +1 -1
  50. package/package.json +27 -17
  51. package/src/checked-prune.ts +27 -0
  52. package/src/exchange-heads.ts +2018 -41
  53. package/src/index.ts +13114 -3167
  54. package/src/pid.ts +24 -13
  55. package/src/ranges.ts +90 -40
  56. package/src/sync/index.ts +115 -2
  57. package/src/sync/rateless-iblt.ts +154 -75
  58. package/src/sync/simple.ts +590 -95
  59. package/src/utils.ts +73 -3
@@ -0,0 +1,816 @@
1
+ // Focused receive/prune hot-path benchmark.
2
+ //
3
+ // Run from packages/programs/data/shared-log:
4
+ // RECEIVE_PRUNE_COUNTS=100,1000,5000 RECEIVE_PRUNE_WARMUP_RUNS=1 RECEIVE_PRUNE_RUNS=1 BENCH_JSON=1 pnpm run benchmark:receive-prune
5
+ // RECEIVE_PRUNE_SCENARIOS=raw-receive-native,raw-receive-native-backbone,raw-receive-native-coordinate-wal,raw-receive-native-backbone-replicating,raw-receive-native-coordinate-wal-replicating,raw-receive-native-coordinate-wal-replicating-defer-verify,raw-receive-native-coordinate-wal-half,raw-receive-native-coordinate-wal-verify-prepare,raw-receive-native-backbone-select-all,raw-receive-native-coordinate-wal-select-all,raw-receive-native-backbone-select-half,raw-receive-native-coordinate-wal-select-half,raw-receive-native-backbone-drop,raw-receive-native-backbone-drop-verify-prepare,request-prune-native-confirm,request-prune-native-backbone-confirm RECEIVE_PRUNE_COUNTS=1000 pnpm run benchmark:receive-prune
6
+ import { deserialize, serialize } from "@dao-xyz/borsh";
7
+ import { DecryptedThing, PreHash } from "@peerbit/crypto";
8
+ import { create as createRustIndexer } from "@peerbit/indexer-rust";
9
+ import { NativeBackboneCoordinatePersistence, NativeBackboneMemoryCoordinatePersistenceStore, createNativeWireSyncSession, } from "@peerbit/native-backbone";
10
+ import { PubSubData, PubSubMessage } from "@peerbit/pubsub-interface";
11
+ import { RPCMessage, RequestV0 } from "@peerbit/rpc";
12
+ import { DataMessage, Message, MessageHeader, SilentDelivery, } from "@peerbit/stream-interface";
13
+ import { TestSession } from "@peerbit/test-utils";
14
+ import { performance } from "node:perf_hooks";
15
+ import { v4 as uuid } from "uuid";
16
+ import { RequestIPrune, createRawExchangeHeadsMessages, } from "../src/exchange-heads.js";
17
+ import { TransportMessage } from "../src/message.js";
18
+ import { createReplicationDomainHash } from "../src/replication-domain-hash.js";
19
+ import { SimpleSyncronizer } from "../src/sync/simple.js";
20
+ import { EventStore } from "../test/utils/stores/event-store.js";
21
+ const parsePositiveInteger = (value, fallback) => {
22
+ if (!value) {
23
+ return fallback;
24
+ }
25
+ const parsed = Number.parseInt(value, 10);
26
+ if (!Number.isFinite(parsed) || parsed <= 0) {
27
+ throw new Error(`Expected a positive integer, got '${value}'`);
28
+ }
29
+ return parsed;
30
+ };
31
+ const parseNonNegativeInteger = (value, fallback) => {
32
+ if (!value) {
33
+ return fallback;
34
+ }
35
+ const parsed = Number.parseInt(value, 10);
36
+ if (!Number.isFinite(parsed) || parsed < 0) {
37
+ throw new Error(`Expected a non-negative integer, got '${value}'`);
38
+ }
39
+ return parsed;
40
+ };
41
+ const parseCounts = (value) => (value ?? "100,1000,5000")
42
+ .split(",")
43
+ .map((part) => part.trim())
44
+ .filter(Boolean)
45
+ .map((part) => parsePositiveInteger(part, 0));
46
+ const parseScenarios = (value) => {
47
+ const scenarios = (value ?? [
48
+ "raw-receive-native",
49
+ "raw-receive-native-backbone",
50
+ "raw-receive-native-coordinate-wal",
51
+ "raw-receive-native-backbone-verify-prepare",
52
+ "raw-receive-native-coordinate-wal-verify-prepare",
53
+ "raw-receive-native-backbone-replicating",
54
+ "raw-receive-native-backbone-replicating-verify-prepare",
55
+ "raw-receive-native-coordinate-wal-replicating",
56
+ "raw-receive-native-coordinate-wal-replicating-verify-prepare",
57
+ "raw-receive-native-backbone-drop",
58
+ "raw-receive-native-backbone-drop-verify-prepare",
59
+ "raw-receive-native-coordinate-wal-drop",
60
+ "raw-receive-native-coordinate-wal-drop-verify-prepare",
61
+ "request-prune-native-confirm",
62
+ "request-prune-native-backbone-confirm",
63
+ "request-prune-native-backbone-coordinate-wal-confirm",
64
+ "request-prune-pending-ihave",
65
+ ].join(","))
66
+ .split(",")
67
+ .map((part) => part.trim())
68
+ .filter(Boolean);
69
+ for (const scenario of scenarios) {
70
+ if (scenario !== "raw-receive-native" &&
71
+ scenario !== "raw-receive-native-backbone" &&
72
+ scenario !== "raw-receive-native-coordinate-wal" &&
73
+ scenario !== "raw-receive-native-backbone-verify-prepare" &&
74
+ scenario !== "raw-receive-native-coordinate-wal-verify-prepare" &&
75
+ scenario !== "raw-receive-native-backbone-replicating" &&
76
+ scenario !==
77
+ "raw-receive-native-backbone-replicating-defer-verify" &&
78
+ scenario !==
79
+ "raw-receive-native-backbone-replicating-verify-prepare" &&
80
+ scenario !== "raw-receive-native-coordinate-wal-replicating" &&
81
+ scenario !==
82
+ "raw-receive-native-coordinate-wal-replicating-defer-verify" &&
83
+ scenario !==
84
+ "raw-receive-native-coordinate-wal-replicating-verify-prepare" &&
85
+ scenario !== "raw-receive-native-backbone-half" &&
86
+ scenario !== "raw-receive-native-backbone-half-verify-prepare" &&
87
+ scenario !== "raw-receive-native-coordinate-wal-half" &&
88
+ scenario !== "raw-receive-native-coordinate-wal-half-verify-prepare" &&
89
+ scenario !== "raw-receive-native-backbone-select-all" &&
90
+ scenario !== "raw-receive-native-coordinate-wal-select-all" &&
91
+ scenario !== "raw-receive-native-backbone-select-half" &&
92
+ scenario !== "raw-receive-native-coordinate-wal-select-half" &&
93
+ scenario !== "raw-receive-native-backbone-drop" &&
94
+ scenario !== "raw-receive-native-backbone-drop-verify-prepare" &&
95
+ scenario !== "raw-receive-native-coordinate-wal-drop" &&
96
+ scenario !== "raw-receive-native-coordinate-wal-drop-verify-prepare" &&
97
+ scenario !== "raw-receive-native-backbone-wire" &&
98
+ scenario !== "raw-receive-native-backbone-wire-fused" &&
99
+ scenario !== "request-prune-native-confirm" &&
100
+ scenario !== "request-prune-native-backbone-confirm" &&
101
+ scenario !== "request-prune-native-backbone-coordinate-wal-confirm" &&
102
+ scenario !== "request-prune-pending-ihave") {
103
+ throw new Error(`Unknown receive/prune scenario '${scenario}'`);
104
+ }
105
+ }
106
+ return scenarios;
107
+ };
108
+ const counts = parseCounts(process.env.RECEIVE_PRUNE_COUNTS);
109
+ const runs = parsePositiveInteger(process.env.RECEIVE_PRUNE_RUNS, 1);
110
+ const warmupRuns = parseNonNegativeInteger(process.env.RECEIVE_PRUNE_WARMUP_RUNS, 0);
111
+ const scenarios = parseScenarios(process.env.RECEIVE_PRUNE_SCENARIOS);
112
+ const setup = {
113
+ domain: createReplicationDomainHash("u32"),
114
+ type: "u32",
115
+ syncronizer: SimpleSyncronizer,
116
+ name: "u32-simple-receive-prune",
117
+ };
118
+ const summarizeProfileEvents = (events) => {
119
+ const summaries = new Map();
120
+ for (const event of events) {
121
+ let summary = summaries.get(event.name);
122
+ if (!summary) {
123
+ summary = {
124
+ name: event.name,
125
+ count: 0,
126
+ totalMs: 0,
127
+ maxMs: 0,
128
+ entries: 0,
129
+ messages: 0,
130
+ nativeBackboneOnly: 0,
131
+ };
132
+ summaries.set(event.name, summary);
133
+ }
134
+ const durationMs = event.durationMs ?? 0;
135
+ summary.count += 1;
136
+ summary.totalMs += durationMs;
137
+ summary.maxMs = Math.max(summary.maxMs, durationMs);
138
+ summary.entries += event.entries ?? 0;
139
+ summary.messages += event.messages ?? 0;
140
+ summary.nativeBackboneOnly +=
141
+ typeof event.details?.nativeBackboneOnly === "number"
142
+ ? event.details.nativeBackboneOnly
143
+ : 0;
144
+ }
145
+ return [...summaries.values()].sort((a, b) => b.totalMs - a.totalMs);
146
+ };
147
+ const roundProfileMetric = (value) => Math.round(value * 1000) / 1000;
148
+ const summarizeProfileSamples = (samples) => {
149
+ if (samples.length === 0) {
150
+ return [];
151
+ }
152
+ const summaries = new Map();
153
+ for (const sample of samples) {
154
+ for (const event of sample.profile) {
155
+ let summary = summaries.get(event.name);
156
+ if (!summary) {
157
+ summary = {
158
+ name: event.name,
159
+ count: 0,
160
+ totalMs: 0,
161
+ maxMs: 0,
162
+ entries: 0,
163
+ messages: 0,
164
+ nativeBackboneOnly: 0,
165
+ };
166
+ summaries.set(event.name, summary);
167
+ }
168
+ summary.count += event.count;
169
+ summary.totalMs += event.totalMs;
170
+ summary.maxMs = Math.max(summary.maxMs, event.maxMs);
171
+ summary.entries += event.entries;
172
+ summary.messages += event.messages;
173
+ summary.nativeBackboneOnly += event.nativeBackboneOnly;
174
+ }
175
+ }
176
+ const divisor = samples.length;
177
+ return [...summaries.values()]
178
+ .map((summary) => ({
179
+ ...summary,
180
+ count: roundProfileMetric(summary.count / divisor),
181
+ totalMs: roundProfileMetric(summary.totalMs / divisor),
182
+ entries: roundProfileMetric(summary.entries / divisor),
183
+ messages: roundProfileMetric(summary.messages / divisor),
184
+ nativeBackboneOnly: roundProfileMetric(summary.nativeBackboneOnly / divisor),
185
+ }))
186
+ .sort((a, b) => b.totalMs - a.totalMs);
187
+ };
188
+ const createOpenArgs = (profileEvents, options) => {
189
+ const nativeBackbone = options?.nativeBackbone || options?.coordinateWal
190
+ ? {
191
+ optional: false,
192
+ ...(options?.coordinateWal
193
+ ? {
194
+ coordinatePersistence: new NativeBackboneCoordinatePersistence(new NativeBackboneMemoryCoordinatePersistenceStore()),
195
+ }
196
+ : {}),
197
+ }
198
+ : undefined;
199
+ return {
200
+ replicate: options?.replicating ? { factor: 1 } : false,
201
+ setup,
202
+ nativeGraph: true,
203
+ nativeBackbone,
204
+ keep: options?.keepHashes
205
+ ? (entry) => options.keepHashes.has(entry.hash)
206
+ : options?.drop || options?.nativeSelectEvery
207
+ ? undefined
208
+ : () => true,
209
+ timeUntilRoleMaturity: 0,
210
+ respondToIHaveTimeout: 1,
211
+ sync: {
212
+ rawExchangeHeads: true,
213
+ ...(options?.verifySignaturesDuringPrepare === undefined
214
+ ? {}
215
+ : {
216
+ rawExchangeHeadsVerifySignaturesDuringPrepare: options.verifySignaturesDuringPrepare,
217
+ }),
218
+ ...(options?.nativeWireSync
219
+ ? { nativeWireSync: options.nativeWireSync }
220
+ : {}),
221
+ profile: (event) => profileEvents.push(event),
222
+ },
223
+ };
224
+ };
225
+ const appendIndependentEntryInfos = async (store, count) => {
226
+ const entries = [];
227
+ for (let i = 0; i < count; i++) {
228
+ const { entry } = await store.add(uuid(), {
229
+ meta: {
230
+ next: [],
231
+ gidSeed: new Uint8Array([i & 0xff, (i >>> 8) & 0xff]),
232
+ },
233
+ replicate: false,
234
+ target: "none",
235
+ });
236
+ entries.push({ hash: entry.hash, gid: entry.meta.gid });
237
+ }
238
+ return entries;
239
+ };
240
+ const appendIndependentEntries = async (store, count) => (await appendIndependentEntryInfos(store, count)).map((entry) => entry.hash);
241
+ const createRawMessages = async (store, hashes) => {
242
+ const messages = [];
243
+ for await (const message of createRawExchangeHeadsMessages(store.log.log, hashes)) {
244
+ messages.push(message);
245
+ }
246
+ return messages;
247
+ };
248
+ const integerBigInt = (value) => typeof value === "bigint" ? value : BigInt(String(value));
249
+ const seedNativeSelectionRanges = (store, entries, retainedPeerHash, keepEvery) => {
250
+ const backbone = store.log._nativeBackbone;
251
+ if (!backbone) {
252
+ throw new Error("Expected native backbone for native selection bench");
253
+ }
254
+ const retainedHashes = new Set();
255
+ for (let i = 0; i < entries.length; i++) {
256
+ const entry = entries[i];
257
+ const retained = i % keepEvery === 0;
258
+ if (retained) {
259
+ retainedHashes.add(entry.hash);
260
+ }
261
+ // Raw entries in this benchmark currently request two replicas; seed both
262
+ // coordinates so dropped entries do not retain through fallback leaders.
263
+ for (const [coordinateIndex, coordinate] of backbone
264
+ .getGidCoordinates(entry.gid, 2)
265
+ .entries()) {
266
+ const start = integerBigInt(coordinate);
267
+ const end = start + 1n;
268
+ backbone.putRange({
269
+ id: `native-select-${i}-${coordinateIndex}`,
270
+ hash: retained
271
+ ? retainedPeerHash
272
+ : `native-select-other-${i}-${coordinateIndex}`,
273
+ timestamp: 0,
274
+ start1: start,
275
+ end1: end,
276
+ start2: start,
277
+ end2: end,
278
+ width: 1,
279
+ mode: 0,
280
+ });
281
+ }
282
+ }
283
+ return retainedHashes;
284
+ };
285
+ const runRawReceive = async (count, run, options) => {
286
+ const session = await TestSession.disconnected(2, {
287
+ indexer: (directory) => createRustIndexer(directory),
288
+ });
289
+ const profileEvents = [];
290
+ const keepHashes = options?.keepEvery ? new Set() : undefined;
291
+ try {
292
+ const wireSession = options?.wire || options?.wireFused
293
+ ? await createNativeWireSyncSession({
294
+ selfHash: session.peers[1].identity.publicKey.hashcode(),
295
+ })
296
+ : undefined;
297
+ const db1 = await session.peers[0].open(new EventStore(), {
298
+ args: createOpenArgs([]),
299
+ });
300
+ const db2 = await session.peers[1].open(new EventStore(), {
301
+ args: createOpenArgs(profileEvents, {
302
+ ...options,
303
+ keepHashes,
304
+ // The unfused wire leg keeps a topicless session: identical
305
+ // native decode+verify, nothing ever stashed.
306
+ nativeWireSync: options?.wireFused ? wireSession : undefined,
307
+ }),
308
+ });
309
+ const entryInfos = await appendIndependentEntryInfos(db1, count);
310
+ const hashes = entryInfos.map((entry) => entry.hash);
311
+ let nativeSelectedHashes;
312
+ if (keepHashes) {
313
+ for (let i = 0; i < hashes.length; i++) {
314
+ if (i % options.keepEvery === 0) {
315
+ keepHashes.add(hashes[i]);
316
+ }
317
+ }
318
+ }
319
+ if (options?.nativeSelectEvery) {
320
+ nativeSelectedHashes = seedNativeSelectionRanges(db2, entryInfos, db2.node.identity.publicKey.hashcode(), options.nativeSelectEvery);
321
+ }
322
+ const messages = await createRawMessages(db1, hashes);
323
+ let frames;
324
+ if (wireSession) {
325
+ const topic = db2.log.topic;
326
+ const receiverHash = db2.node.identity.publicKey.hashcode();
327
+ const sign = (bytes) => db1.node.identity.sign(bytes, PreHash.SHA_256);
328
+ frames = [];
329
+ for (const message of messages) {
330
+ const requestBytes = serialize(new RequestV0({
331
+ request: new DecryptedThing({
332
+ data: serialize(message),
333
+ }),
334
+ }));
335
+ const pubsubBytes = serialize(new PubSubData({ topics: [topic], data: requestBytes, strict: true }));
336
+ const dataMessage = await new DataMessage({
337
+ header: new MessageHeader({
338
+ session: 0,
339
+ mode: new SilentDelivery({ to: [receiverHash], redundancy: 1 }),
340
+ }),
341
+ data: pubsubBytes,
342
+ }).sign(sign);
343
+ const bytes = dataMessage.bytes();
344
+ frames.push(bytes instanceof Uint8Array ? bytes : bytes.subarray());
345
+ }
346
+ }
347
+ const started = performance.now();
348
+ if (wireSession && frames) {
349
+ for (const frame of frames) {
350
+ const records = wireSession.decodeAndVerifyBatch([frame], Date.now());
351
+ if ((records[0] & 0x01) === 0) {
352
+ throw new Error("Wire frame decode failed");
353
+ }
354
+ // Shared with both legs: the envelope + payload shell decodes
355
+ // the stream/pubsub/rpc layers perform either way.
356
+ const envelope = deserialize(frame, Message);
357
+ const pubsubMessage = deserialize(envelope.data, PubSubMessage);
358
+ const rpcMessage = deserialize(pubsubMessage.data, RPCMessage);
359
+ let received;
360
+ if (options?.wireFused) {
361
+ const resolved = db2.log.resolveStashedRawExchangeHeadsMessage(envelope);
362
+ if (!resolved) {
363
+ throw new Error("Expected a fused wire stash resolve");
364
+ }
365
+ received = resolved;
366
+ }
367
+ else {
368
+ const decrypted = await rpcMessage.request.decrypt();
369
+ received = decrypted.getValue(TransportMessage);
370
+ }
371
+ await db2.log.onMessage(received, {
372
+ from: db1.node.identity.publicKey,
373
+ });
374
+ }
375
+ }
376
+ else {
377
+ for (const message of messages) {
378
+ await db2.log.onMessage(message, {
379
+ from: db1.node.identity.publicKey,
380
+ });
381
+ }
382
+ }
383
+ const elapsed = performance.now() - started;
384
+ const expectedReceived = options?.drop
385
+ ? 0
386
+ : nativeSelectedHashes
387
+ ? nativeSelectedHashes.size
388
+ : keepHashes
389
+ ? keepHashes.size
390
+ : count;
391
+ if (db2.log.log.length !== expectedReceived) {
392
+ throw new Error(`Expected ${expectedReceived} raw received entries, got ${db2.log.log.length}`);
393
+ }
394
+ if (options?.nativeSelectEvery) {
395
+ const expectedDropped = count - expectedReceived;
396
+ const nativeSelectEvents = profileEvents.filter((event) => event.name === "sharedLog.rawReceive.nativeSelect");
397
+ if (nativeSelectEvents.length === 0 && expectedDropped > 0) {
398
+ throw new Error("Expected native raw receive selection profile event");
399
+ }
400
+ const retainedByNativeSelect = nativeSelectEvents.reduce((sum, event) => sum + (event.count ?? 0), 0);
401
+ const droppedByNativeSelect = nativeSelectEvents.reduce((sum, event) => sum +
402
+ (typeof event.details?.dropped === "number"
403
+ ? event.details.dropped
404
+ : 0), 0);
405
+ if (expectedDropped > 0 &&
406
+ retainedByNativeSelect !== expectedReceived) {
407
+ throw new Error(`Expected native raw receive selection to retain ${expectedReceived} entries, retained ${retainedByNativeSelect}`);
408
+ }
409
+ if (droppedByNativeSelect !== expectedDropped) {
410
+ throw new Error(`Expected native raw receive selection to drop ${expectedDropped} entries, dropped ${droppedByNativeSelect}`);
411
+ }
412
+ }
413
+ return {
414
+ scenario: options?.wireFused
415
+ ? "raw-receive-native-backbone-wire-fused"
416
+ : options?.wire
417
+ ? "raw-receive-native-backbone-wire"
418
+ : options?.coordinateWal &&
419
+ options.drop &&
420
+ options.verifySignaturesDuringPrepare === true
421
+ ? "raw-receive-native-coordinate-wal-drop-verify-prepare"
422
+ : options?.nativeBackbone &&
423
+ options.drop &&
424
+ options.verifySignaturesDuringPrepare === true
425
+ ? "raw-receive-native-backbone-drop-verify-prepare"
426
+ : options?.coordinateWal && options.drop
427
+ ? "raw-receive-native-coordinate-wal-drop"
428
+ : options?.nativeBackbone && options.drop
429
+ ? "raw-receive-native-backbone-drop"
430
+ : options?.coordinateWal &&
431
+ options.keepEvery === 2 &&
432
+ options.verifySignaturesDuringPrepare === true
433
+ ? "raw-receive-native-coordinate-wal-half-verify-prepare"
434
+ : options?.nativeBackbone &&
435
+ options.keepEvery === 2 &&
436
+ options.verifySignaturesDuringPrepare === true
437
+ ? "raw-receive-native-backbone-half-verify-prepare"
438
+ : options?.coordinateWal && options.keepEvery === 2
439
+ ? "raw-receive-native-coordinate-wal-half"
440
+ : options?.nativeBackbone && options.keepEvery === 2
441
+ ? "raw-receive-native-backbone-half"
442
+ : options?.coordinateWal && options.nativeSelectEvery === 1
443
+ ? "raw-receive-native-coordinate-wal-select-all"
444
+ : options?.nativeBackbone && options.nativeSelectEvery === 1
445
+ ? "raw-receive-native-backbone-select-all"
446
+ : options?.coordinateWal && options.nativeSelectEvery === 2
447
+ ? "raw-receive-native-coordinate-wal-select-half"
448
+ : options?.nativeBackbone && options.nativeSelectEvery === 2
449
+ ? "raw-receive-native-backbone-select-half"
450
+ : options?.coordinateWal &&
451
+ options.replicating &&
452
+ options.verifySignaturesDuringPrepare === false
453
+ ? "raw-receive-native-coordinate-wal-replicating-defer-verify"
454
+ : options?.nativeBackbone &&
455
+ options.replicating &&
456
+ options.verifySignaturesDuringPrepare === false
457
+ ? "raw-receive-native-backbone-replicating-defer-verify"
458
+ : options?.coordinateWal &&
459
+ options.replicating &&
460
+ options.verifySignaturesDuringPrepare === true
461
+ ? "raw-receive-native-coordinate-wal-replicating-verify-prepare"
462
+ : options?.nativeBackbone &&
463
+ options.replicating &&
464
+ options.verifySignaturesDuringPrepare === true
465
+ ? "raw-receive-native-backbone-replicating-verify-prepare"
466
+ : options?.coordinateWal && options.replicating
467
+ ? "raw-receive-native-coordinate-wal-replicating"
468
+ : options?.nativeBackbone && options.replicating
469
+ ? "raw-receive-native-backbone-replicating"
470
+ : options?.coordinateWal &&
471
+ options.verifySignaturesDuringPrepare === true
472
+ ? "raw-receive-native-coordinate-wal-verify-prepare"
473
+ : options?.nativeBackbone &&
474
+ options.verifySignaturesDuringPrepare === true
475
+ ? "raw-receive-native-backbone-verify-prepare"
476
+ : options?.coordinateWal
477
+ ? "raw-receive-native-coordinate-wal"
478
+ : options?.nativeBackbone
479
+ ? "raw-receive-native-backbone"
480
+ : "raw-receive-native",
481
+ count,
482
+ run,
483
+ elapsedMs: elapsed,
484
+ opsPerSecond: (count / elapsed) * 1000,
485
+ profile: summarizeProfileEvents(profileEvents),
486
+ };
487
+ }
488
+ finally {
489
+ await session.stop();
490
+ }
491
+ };
492
+ const installFastLeaderWaits = (store) => {
493
+ const log = store.log;
494
+ const selfHash = store.node.identity.publicKey.hashcode();
495
+ const originalWaitForGid = log._waitForGidReplicators;
496
+ const originalWaitForEntry = log._waitForEntryReplicators;
497
+ log._waitForGidReplicators = async (_gid, _replicas, _waitFor, options) => {
498
+ options?.onLeader?.(selfHash);
499
+ return new Map([[selfHash, { intersecting: true }]]);
500
+ };
501
+ log._waitForEntryReplicators = async (_entry, _replicas, _waitFor, options) => {
502
+ options?.onLeader?.(selfHash);
503
+ return new Map([[selfHash, { intersecting: true }]]);
504
+ };
505
+ return () => {
506
+ log._waitForGidReplicators = originalWaitForGid;
507
+ log._waitForEntryReplicators = originalWaitForEntry;
508
+ };
509
+ };
510
+ const suppressPruneResponses = (store) => {
511
+ const debounced = store.log.responseToPruneDebouncedFn;
512
+ const originalAdd = debounced.add;
513
+ debounced.add = () => undefined;
514
+ return () => {
515
+ debounced.add = originalAdd;
516
+ };
517
+ };
518
+ const clearPendingIHaves = (store) => {
519
+ const pending = store.log._pendingIHave;
520
+ for (const value of pending.values()) {
521
+ value.clear?.();
522
+ }
523
+ pending.clear();
524
+ };
525
+ const runRequestPruneNativeConfirm = async (count, run, options) => {
526
+ const session = await TestSession.disconnected(2, {
527
+ indexer: (directory) => createRustIndexer(directory),
528
+ });
529
+ const profileEvents = [];
530
+ try {
531
+ const db = await session.peers[1].open(new EventStore(), {
532
+ args: {
533
+ ...createOpenArgs(profileEvents, options),
534
+ replicate: { factor: 1 },
535
+ },
536
+ });
537
+ const hashes = await appendIndependentEntries(db, count);
538
+ const restoreWaits = installFastLeaderWaits(db);
539
+ const restoreResponses = suppressPruneResponses(db);
540
+ try {
541
+ const started = performance.now();
542
+ await db.log.onMessage(new RequestIPrune({ hashes }), {
543
+ from: session.peers[0].identity.publicKey,
544
+ });
545
+ const elapsed = performance.now() - started;
546
+ return {
547
+ scenario: options?.coordinateWal
548
+ ? "request-prune-native-backbone-coordinate-wal-confirm"
549
+ : options?.nativeBackbone
550
+ ? "request-prune-native-backbone-confirm"
551
+ : "request-prune-native-confirm",
552
+ count,
553
+ run,
554
+ elapsedMs: elapsed,
555
+ opsPerSecond: (count / elapsed) * 1000,
556
+ profile: summarizeProfileEvents(profileEvents),
557
+ };
558
+ }
559
+ finally {
560
+ restoreResponses();
561
+ restoreWaits();
562
+ clearPendingIHaves(db);
563
+ }
564
+ }
565
+ finally {
566
+ await session.stop();
567
+ }
568
+ };
569
+ const runRequestPrunePendingIHave = async (count, run) => {
570
+ const session = await TestSession.disconnected(2, {
571
+ indexer: (directory) => createRustIndexer(directory),
572
+ });
573
+ const sourceProfileEvents = [];
574
+ const targetProfileEvents = [];
575
+ try {
576
+ const source = await session.peers[0].open(new EventStore(), {
577
+ args: createOpenArgs(sourceProfileEvents),
578
+ });
579
+ const target = await session.peers[1].open(new EventStore(), {
580
+ args: createOpenArgs(targetProfileEvents),
581
+ });
582
+ const hashes = await appendIndependentEntries(source, count);
583
+ try {
584
+ const started = performance.now();
585
+ await target.log.onMessage(new RequestIPrune({ hashes }), {
586
+ from: source.node.identity.publicKey,
587
+ });
588
+ const elapsed = performance.now() - started;
589
+ return {
590
+ scenario: "request-prune-pending-ihave",
591
+ count,
592
+ run,
593
+ elapsedMs: elapsed,
594
+ opsPerSecond: (count / elapsed) * 1000,
595
+ profile: summarizeProfileEvents(targetProfileEvents),
596
+ };
597
+ }
598
+ finally {
599
+ clearPendingIHaves(target);
600
+ }
601
+ }
602
+ finally {
603
+ await session.stop();
604
+ }
605
+ };
606
+ const runScenario = async (scenario, count, run) => {
607
+ if (scenario === "raw-receive-native") {
608
+ return runRawReceive(count, run);
609
+ }
610
+ if (scenario === "raw-receive-native-backbone") {
611
+ return runRawReceive(count, run, { nativeBackbone: true });
612
+ }
613
+ if (scenario === "raw-receive-native-coordinate-wal") {
614
+ return runRawReceive(count, run, { coordinateWal: true });
615
+ }
616
+ if (scenario === "raw-receive-native-backbone-verify-prepare") {
617
+ return runRawReceive(count, run, {
618
+ nativeBackbone: true,
619
+ verifySignaturesDuringPrepare: true,
620
+ });
621
+ }
622
+ if (scenario === "raw-receive-native-coordinate-wal-verify-prepare") {
623
+ return runRawReceive(count, run, {
624
+ coordinateWal: true,
625
+ verifySignaturesDuringPrepare: true,
626
+ });
627
+ }
628
+ if (scenario === "raw-receive-native-backbone-replicating") {
629
+ return runRawReceive(count, run, {
630
+ nativeBackbone: true,
631
+ replicating: true,
632
+ });
633
+ }
634
+ if (scenario === "raw-receive-native-backbone-replicating-defer-verify") {
635
+ return runRawReceive(count, run, {
636
+ nativeBackbone: true,
637
+ replicating: true,
638
+ verifySignaturesDuringPrepare: false,
639
+ });
640
+ }
641
+ if (scenario === "raw-receive-native-backbone-replicating-verify-prepare") {
642
+ return runRawReceive(count, run, {
643
+ nativeBackbone: true,
644
+ replicating: true,
645
+ verifySignaturesDuringPrepare: true,
646
+ });
647
+ }
648
+ if (scenario === "raw-receive-native-coordinate-wal-replicating") {
649
+ return runRawReceive(count, run, {
650
+ coordinateWal: true,
651
+ replicating: true,
652
+ });
653
+ }
654
+ if (scenario === "raw-receive-native-coordinate-wal-replicating-defer-verify") {
655
+ return runRawReceive(count, run, {
656
+ coordinateWal: true,
657
+ replicating: true,
658
+ verifySignaturesDuringPrepare: false,
659
+ });
660
+ }
661
+ if (scenario === "raw-receive-native-coordinate-wal-replicating-verify-prepare") {
662
+ return runRawReceive(count, run, {
663
+ coordinateWal: true,
664
+ replicating: true,
665
+ verifySignaturesDuringPrepare: true,
666
+ });
667
+ }
668
+ if (scenario === "raw-receive-native-backbone-half") {
669
+ return runRawReceive(count, run, {
670
+ nativeBackbone: true,
671
+ keepEvery: 2,
672
+ });
673
+ }
674
+ if (scenario === "raw-receive-native-backbone-half-verify-prepare") {
675
+ return runRawReceive(count, run, {
676
+ nativeBackbone: true,
677
+ keepEvery: 2,
678
+ verifySignaturesDuringPrepare: true,
679
+ });
680
+ }
681
+ if (scenario === "raw-receive-native-coordinate-wal-half") {
682
+ return runRawReceive(count, run, {
683
+ coordinateWal: true,
684
+ keepEvery: 2,
685
+ });
686
+ }
687
+ if (scenario === "raw-receive-native-coordinate-wal-half-verify-prepare") {
688
+ return runRawReceive(count, run, {
689
+ coordinateWal: true,
690
+ keepEvery: 2,
691
+ verifySignaturesDuringPrepare: true,
692
+ });
693
+ }
694
+ if (scenario === "raw-receive-native-backbone-select-half") {
695
+ return runRawReceive(count, run, {
696
+ nativeBackbone: true,
697
+ nativeSelectEvery: 2,
698
+ });
699
+ }
700
+ if (scenario === "raw-receive-native-coordinate-wal-select-half") {
701
+ return runRawReceive(count, run, {
702
+ coordinateWal: true,
703
+ nativeSelectEvery: 2,
704
+ });
705
+ }
706
+ if (scenario === "raw-receive-native-backbone-select-all") {
707
+ return runRawReceive(count, run, {
708
+ nativeBackbone: true,
709
+ nativeSelectEvery: 1,
710
+ });
711
+ }
712
+ if (scenario === "raw-receive-native-coordinate-wal-select-all") {
713
+ return runRawReceive(count, run, {
714
+ coordinateWal: true,
715
+ nativeSelectEvery: 1,
716
+ });
717
+ }
718
+ if (scenario === "raw-receive-native-backbone-drop") {
719
+ return runRawReceive(count, run, {
720
+ nativeBackbone: true,
721
+ drop: true,
722
+ });
723
+ }
724
+ if (scenario === "raw-receive-native-backbone-drop-verify-prepare") {
725
+ return runRawReceive(count, run, {
726
+ nativeBackbone: true,
727
+ drop: true,
728
+ verifySignaturesDuringPrepare: true,
729
+ });
730
+ }
731
+ if (scenario === "raw-receive-native-coordinate-wal-drop") {
732
+ return runRawReceive(count, run, {
733
+ coordinateWal: true,
734
+ drop: true,
735
+ });
736
+ }
737
+ if (scenario === "raw-receive-native-coordinate-wal-drop-verify-prepare") {
738
+ return runRawReceive(count, run, {
739
+ coordinateWal: true,
740
+ drop: true,
741
+ verifySignaturesDuringPrepare: true,
742
+ });
743
+ }
744
+ if (scenario === "raw-receive-native-backbone-wire") {
745
+ return runRawReceive(count, run, {
746
+ nativeBackbone: true,
747
+ wire: true,
748
+ });
749
+ }
750
+ if (scenario === "raw-receive-native-backbone-wire-fused") {
751
+ return runRawReceive(count, run, {
752
+ nativeBackbone: true,
753
+ wire: true,
754
+ wireFused: true,
755
+ });
756
+ }
757
+ if (scenario === "request-prune-native-confirm") {
758
+ return runRequestPruneNativeConfirm(count, run);
759
+ }
760
+ if (scenario === "request-prune-native-backbone-confirm") {
761
+ return runRequestPruneNativeConfirm(count, run, {
762
+ nativeBackbone: true,
763
+ });
764
+ }
765
+ if (scenario === "request-prune-native-backbone-coordinate-wal-confirm") {
766
+ return runRequestPruneNativeConfirm(count, run, {
767
+ coordinateWal: true,
768
+ });
769
+ }
770
+ return runRequestPrunePendingIHave(count, run);
771
+ };
772
+ const rows = [];
773
+ for (const scenario of scenarios) {
774
+ for (const count of counts) {
775
+ for (let warmup = 0; warmup < warmupRuns; warmup++) {
776
+ await runScenario(scenario, count, -1 - warmup);
777
+ }
778
+ for (let run = 0; run < runs; run++) {
779
+ rows.push(await runScenario(scenario, count, run));
780
+ }
781
+ }
782
+ }
783
+ const aggregateRows = [...new Set(rows.map((row) => row.scenario))].flatMap((scenario) => counts.map((count) => {
784
+ const samples = rows.filter((row) => row.scenario === scenario && row.count === count);
785
+ const meanMs = samples.reduce((sum, row) => sum + row.elapsedMs, 0) / samples.length;
786
+ const meanOps = samples.reduce((sum, row) => sum + row.opsPerSecond, 0) /
787
+ samples.length;
788
+ return {
789
+ scenario,
790
+ count,
791
+ runs: samples.length,
792
+ meanMs: Math.round(meanMs * 100) / 100,
793
+ meanOpsPerSecond: Math.round(meanOps),
794
+ profile: summarizeProfileSamples(samples),
795
+ };
796
+ }));
797
+ if (process.env.BENCH_JSON === "1") {
798
+ process.stdout.write(JSON.stringify({
799
+ name: "shared-log-receive-prune",
800
+ warmupRuns,
801
+ rows,
802
+ aggregateRows,
803
+ }, null, 2));
804
+ }
805
+ else {
806
+ console.table(aggregateRows.map((row) => ({
807
+ scenario: row.scenario,
808
+ count: row.count,
809
+ runs: row.runs,
810
+ meanMs: row.meanMs,
811
+ meanOpsPerSecond: row.meanOpsPerSecond,
812
+ })));
813
+ console.dir(aggregateRows, { depth: 6 });
814
+ }
815
+ process.exit(process.exitCode ?? 0);
816
+ //# sourceMappingURL=receive-prune.js.map