@peerbit/shared-log 12.3.5-cb91e7b → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/sync-batch-sweep.d.ts +2 -0
- package/dist/benchmark/sync-batch-sweep.d.ts.map +1 -0
- package/dist/benchmark/sync-batch-sweep.js +305 -0
- package/dist/benchmark/sync-batch-sweep.js.map +1 -0
- package/dist/src/index.d.ts +14 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +458 -67
- package/dist/src/index.js.map +1 -1
- package/dist/src/ranges.d.ts +3 -1
- package/dist/src/ranges.d.ts.map +1 -1
- package/dist/src/ranges.js +7 -2
- package/dist/src/ranges.js.map +1 -1
- package/dist/src/sync/index.d.ts +45 -1
- package/dist/src/sync/index.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.d.ts +13 -2
- package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.js +152 -0
- package/dist/src/sync/rateless-iblt.js.map +1 -1
- package/dist/src/sync/simple.d.ts +24 -3
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +330 -32
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +20 -20
- package/src/index.ts +598 -136
- package/src/ranges.ts +7 -1
- package/src/sync/index.ts +53 -1
- package/src/sync/rateless-iblt.ts +179 -1
- package/src/sync/simple.ts +427 -41
package/dist/src/index.js
CHANGED
|
@@ -216,6 +216,20 @@ const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
|
|
|
216
216
|
const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
|
|
217
217
|
const RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR = 1e-3;
|
|
218
218
|
const DEFAULT_DISTRIBUTION_DEBOUNCE_TIME = 500;
|
|
219
|
+
const RECENT_REPAIR_DISPATCH_TTL_MS = 5_000;
|
|
220
|
+
const REPAIR_SWEEP_ENTRY_BATCH_SIZE = 1_000;
|
|
221
|
+
const REPAIR_SWEEP_TARGET_BUFFER_SIZE = 1024;
|
|
222
|
+
const FORCE_FRESH_RETRY_SCHEDULE_MS = [0, 1_000, 3_000, 7_000];
|
|
223
|
+
const JOIN_WARMUP_RETRY_SCHEDULE_MS = [0, 1_000, 3_000];
|
|
224
|
+
const toPositiveInteger = (value, fallback, label) => {
|
|
225
|
+
if (value == null) {
|
|
226
|
+
return fallback;
|
|
227
|
+
}
|
|
228
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
229
|
+
throw new Error(`${label} must be a positive number`);
|
|
230
|
+
}
|
|
231
|
+
return Math.max(1, Math.floor(value));
|
|
232
|
+
};
|
|
219
233
|
const DEFAULT_SHARED_LOG_FANOUT_CHANNEL_OPTIONS = {
|
|
220
234
|
msgRate: 30,
|
|
221
235
|
msgSize: 1024,
|
|
@@ -305,6 +319,11 @@ let SharedLog = (() => {
|
|
|
305
319
|
_requestIPruneResponseReplicatorSet; // tracks entry hash to peer hash
|
|
306
320
|
_checkedPruneRetries;
|
|
307
321
|
replicationChangeDebounceFn;
|
|
322
|
+
_repairRetryTimers;
|
|
323
|
+
_recentRepairDispatch;
|
|
324
|
+
_repairSweepRunning;
|
|
325
|
+
_repairSweepForceFreshPending;
|
|
326
|
+
_repairSweepAddedPeersPending;
|
|
308
327
|
// regular distribution checks
|
|
309
328
|
distributeQueue;
|
|
310
329
|
syncronizer;
|
|
@@ -316,6 +335,7 @@ let SharedLog = (() => {
|
|
|
316
335
|
waitForReplicatorRequestMaxAttempts;
|
|
317
336
|
waitForPruneDelay;
|
|
318
337
|
distributionDebounceTime;
|
|
338
|
+
repairSweepTargetBufferSize;
|
|
319
339
|
replicationController;
|
|
320
340
|
history;
|
|
321
341
|
domain;
|
|
@@ -486,22 +506,23 @@ let SharedLog = (() => {
|
|
|
486
506
|
const delivery = deliveryArg === undefined || deliveryArg === false
|
|
487
507
|
? undefined
|
|
488
508
|
: deliveryArg === true
|
|
489
|
-
? {}
|
|
509
|
+
? { reliability: "ack" }
|
|
490
510
|
: deliveryArg;
|
|
491
511
|
if (!delivery) {
|
|
492
512
|
return {
|
|
493
513
|
delivery: undefined,
|
|
514
|
+
reliability: "best-effort",
|
|
494
515
|
requireRecipients: false,
|
|
495
|
-
|
|
516
|
+
minAcks: undefined,
|
|
496
517
|
wrap: undefined,
|
|
497
518
|
};
|
|
498
519
|
}
|
|
499
|
-
const
|
|
520
|
+
const reliability = delivery.reliability ?? "ack";
|
|
500
521
|
const deliveryTimeout = delivery.timeout;
|
|
501
522
|
const deliverySignal = delivery.signal;
|
|
502
523
|
const requireRecipients = delivery.requireRecipients === true;
|
|
503
|
-
const
|
|
504
|
-
? Math.max(0, Math.floor(
|
|
524
|
+
const minAcks = delivery.minAcks != null && Number.isFinite(delivery.minAcks)
|
|
525
|
+
? Math.max(0, Math.floor(delivery.minAcks))
|
|
505
526
|
: undefined;
|
|
506
527
|
const wrap = deliveryTimeout == null && deliverySignal == null
|
|
507
528
|
? undefined
|
|
@@ -562,13 +583,83 @@ let SharedLog = (() => {
|
|
|
562
583
|
});
|
|
563
584
|
return {
|
|
564
585
|
delivery,
|
|
586
|
+
reliability,
|
|
565
587
|
requireRecipients,
|
|
566
|
-
|
|
588
|
+
minAcks,
|
|
567
589
|
wrap,
|
|
568
590
|
};
|
|
569
591
|
}
|
|
592
|
+
async _getSortedRouteHints(targetHash) {
|
|
593
|
+
const pubsub = this.node.services.pubsub;
|
|
594
|
+
const maybeHints = await pubsub?.getUnifiedRouteHints?.(this.topic, targetHash);
|
|
595
|
+
const hints = Array.isArray(maybeHints) ? maybeHints : [];
|
|
596
|
+
const now = Date.now();
|
|
597
|
+
return hints
|
|
598
|
+
.filter((hint) => hint.expiresAt == null || hint.expiresAt > now)
|
|
599
|
+
.sort((a, b) => {
|
|
600
|
+
const rankA = a.kind === "directstream-ack" ? 0 : 1;
|
|
601
|
+
const rankB = b.kind === "directstream-ack" ? 0 : 1;
|
|
602
|
+
if (rankA !== rankB) {
|
|
603
|
+
return rankA - rankB;
|
|
604
|
+
}
|
|
605
|
+
const costA = a.kind === "directstream-ack"
|
|
606
|
+
? a.distance
|
|
607
|
+
: Math.max(0, (a.route?.length ?? 1) - 1);
|
|
608
|
+
const costB = b.kind === "directstream-ack"
|
|
609
|
+
? b.distance
|
|
610
|
+
: Math.max(0, (b.route?.length ?? 1) - 1);
|
|
611
|
+
if (costA !== costB) {
|
|
612
|
+
return costA - costB;
|
|
613
|
+
}
|
|
614
|
+
return (b.updatedAt ?? 0) - (a.updatedAt ?? 0);
|
|
615
|
+
});
|
|
616
|
+
}
|
|
617
|
+
async _sendAckWithUnifiedHints(properties) {
|
|
618
|
+
const { peer, message, payload, fanoutUnicastOptions } = properties;
|
|
619
|
+
const hints = await this._getSortedRouteHints(peer);
|
|
620
|
+
const hasDirectHint = hints.some((hint) => hint.kind === "directstream-ack");
|
|
621
|
+
const fanoutHint = hints.find((hint) => hint.kind === "fanout-token");
|
|
622
|
+
if (hasDirectHint) {
|
|
623
|
+
try {
|
|
624
|
+
await this.rpc.send(message, {
|
|
625
|
+
mode: new AcknowledgeDelivery({
|
|
626
|
+
redundancy: 1,
|
|
627
|
+
to: [peer],
|
|
628
|
+
}),
|
|
629
|
+
});
|
|
630
|
+
return;
|
|
631
|
+
}
|
|
632
|
+
catch {
|
|
633
|
+
// Fall back to fanout token/direct fanout unicast below.
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
if (fanoutHint && this._fanoutChannel) {
|
|
637
|
+
try {
|
|
638
|
+
await this._fanoutChannel.unicastAck(fanoutHint.route, payload, fanoutUnicastOptions);
|
|
639
|
+
return;
|
|
640
|
+
}
|
|
641
|
+
catch {
|
|
642
|
+
// Fall back below.
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
if (this._fanoutChannel) {
|
|
646
|
+
try {
|
|
647
|
+
await this._fanoutChannel.unicastToAck(peer, payload, fanoutUnicastOptions);
|
|
648
|
+
return;
|
|
649
|
+
}
|
|
650
|
+
catch {
|
|
651
|
+
// Fall back below.
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
await this.rpc.send(message, {
|
|
655
|
+
mode: new AcknowledgeDelivery({
|
|
656
|
+
redundancy: 1,
|
|
657
|
+
to: [peer],
|
|
658
|
+
}),
|
|
659
|
+
});
|
|
660
|
+
}
|
|
570
661
|
async _appendDeliverToReplicators(entry, minReplicasValue, leaders, selfHash, isLeader, deliveryArg) {
|
|
571
|
-
const { delivery, requireRecipients,
|
|
662
|
+
const { delivery, reliability, requireRecipients, minAcks, wrap } = this._parseDeliveryOptions(deliveryArg);
|
|
572
663
|
const pending = [];
|
|
573
664
|
const track = (promise) => {
|
|
574
665
|
pending.push(wrap ? wrap(promise) : promise);
|
|
@@ -580,7 +671,28 @@ let SharedLog = (() => {
|
|
|
580
671
|
await this._mergeLeadersFromGidReferences(message, minReplicasValue, leaders);
|
|
581
672
|
const leadersForDelivery = delivery ? new Set(leaders.keys()) : undefined;
|
|
582
673
|
const set = this.addPeersToGidPeerHistory(entry.meta.gid, leaders.keys());
|
|
583
|
-
|
|
674
|
+
let hasRemotePeers = set.has(selfHash) ? set.size > 1 : set.size > 0;
|
|
675
|
+
const allowSubscriberFallback = this.syncronizer instanceof SimpleSyncronizer ||
|
|
676
|
+
(this.compatibility ?? Number.MAX_VALUE) < 10;
|
|
677
|
+
if (!hasRemotePeers && allowSubscriberFallback) {
|
|
678
|
+
try {
|
|
679
|
+
const subscribers = await this._getTopicSubscribers(this.topic);
|
|
680
|
+
if (subscribers && subscribers.length > 0) {
|
|
681
|
+
for (const subscriber of subscribers) {
|
|
682
|
+
const hash = subscriber.hashcode();
|
|
683
|
+
if (hash === selfHash) {
|
|
684
|
+
continue;
|
|
685
|
+
}
|
|
686
|
+
set.add(hash);
|
|
687
|
+
leadersForDelivery?.add(hash);
|
|
688
|
+
}
|
|
689
|
+
hasRemotePeers = set.has(selfHash) ? set.size > 1 : set.size > 0;
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
catch {
|
|
693
|
+
// Best-effort only; keep discovered recipients as-is.
|
|
694
|
+
}
|
|
695
|
+
}
|
|
584
696
|
if (!hasRemotePeers) {
|
|
585
697
|
if (requireRecipients) {
|
|
586
698
|
throw new NoPeersError(this.rpc.topic);
|
|
@@ -617,7 +729,9 @@ let SharedLog = (() => {
|
|
|
617
729
|
let silentTo;
|
|
618
730
|
// Default delivery semantics: require enough remote ACKs to reach the requested
|
|
619
731
|
// replication degree (local append counts as 1).
|
|
620
|
-
const
|
|
732
|
+
const defaultMinAcks = Math.max(0, minReplicasValue - 1);
|
|
733
|
+
const ackLimitRaw = reliability === "ack" ? (minAcks ?? defaultMinAcks) : 0;
|
|
734
|
+
const ackLimit = Math.max(0, Math.min(Math.floor(ackLimitRaw), orderedRemoteRecipients.length));
|
|
621
735
|
for (const peer of orderedRemoteRecipients) {
|
|
622
736
|
if (ackTo.length < ackLimit) {
|
|
623
737
|
ackTo.push(peer);
|
|
@@ -637,36 +751,11 @@ let SharedLog = (() => {
|
|
|
637
751
|
const payload = serialize(message);
|
|
638
752
|
for (const peer of ackTo) {
|
|
639
753
|
track((async () => {
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
const canDirectFast = Boolean(pubsub?.peers?.get?.(peer)?.isWritable) ||
|
|
646
|
-
Boolean(pubsub?.routes?.isReachable?.(pubsub?.publicKeyHash, peer, 0));
|
|
647
|
-
if (canDirectFast) {
|
|
648
|
-
await this.rpc.send(message, {
|
|
649
|
-
mode: new AcknowledgeDelivery({
|
|
650
|
-
redundancy: 1,
|
|
651
|
-
to: [peer],
|
|
652
|
-
}),
|
|
653
|
-
});
|
|
654
|
-
return;
|
|
655
|
-
}
|
|
656
|
-
if (this._fanoutChannel) {
|
|
657
|
-
try {
|
|
658
|
-
await this._fanoutChannel.unicastToAck(peer, payload, fanoutUnicastOptions);
|
|
659
|
-
return;
|
|
660
|
-
}
|
|
661
|
-
catch {
|
|
662
|
-
// fall back below
|
|
663
|
-
}
|
|
664
|
-
}
|
|
665
|
-
await this.rpc.send(message, {
|
|
666
|
-
mode: new AcknowledgeDelivery({
|
|
667
|
-
redundancy: 1,
|
|
668
|
-
to: [peer],
|
|
669
|
-
}),
|
|
754
|
+
await this._sendAckWithUnifiedHints({
|
|
755
|
+
peer,
|
|
756
|
+
message,
|
|
757
|
+
payload,
|
|
758
|
+
fanoutUnicastOptions,
|
|
670
759
|
});
|
|
671
760
|
})());
|
|
672
761
|
}
|
|
@@ -1159,6 +1248,13 @@ let SharedLog = (() => {
|
|
|
1159
1248
|
}
|
|
1160
1249
|
this.pendingMaturity.delete(keyHash);
|
|
1161
1250
|
}
|
|
1251
|
+
// Keep local sync/prune state consistent even when a peer disappears
|
|
1252
|
+
// through replication-info updates without a topic unsubscribe event.
|
|
1253
|
+
this.removePeerFromGidPeerHistory(keyHash);
|
|
1254
|
+
this._recentRepairDispatch.delete(keyHash);
|
|
1255
|
+
if (!isMe) {
|
|
1256
|
+
this.syncronizer.onPeerDisconnected(keyHash);
|
|
1257
|
+
}
|
|
1162
1258
|
if (!isMe) {
|
|
1163
1259
|
this.rebalanceParticipationDebounced?.call();
|
|
1164
1260
|
}
|
|
@@ -1522,6 +1618,181 @@ let SharedLog = (() => {
|
|
|
1522
1618
|
}
|
|
1523
1619
|
return set;
|
|
1524
1620
|
}
|
|
1621
|
+
dispatchMaybeMissingEntries(target, entries, options) {
|
|
1622
|
+
if (entries.size === 0) {
|
|
1623
|
+
return;
|
|
1624
|
+
}
|
|
1625
|
+
const now = Date.now();
|
|
1626
|
+
let recentlyDispatchedByHash = this._recentRepairDispatch.get(target);
|
|
1627
|
+
if (!recentlyDispatchedByHash) {
|
|
1628
|
+
recentlyDispatchedByHash = new Map();
|
|
1629
|
+
this._recentRepairDispatch.set(target, recentlyDispatchedByHash);
|
|
1630
|
+
}
|
|
1631
|
+
for (const [hash, ts] of recentlyDispatchedByHash) {
|
|
1632
|
+
if (now - ts > RECENT_REPAIR_DISPATCH_TTL_MS) {
|
|
1633
|
+
recentlyDispatchedByHash.delete(hash);
|
|
1634
|
+
}
|
|
1635
|
+
}
|
|
1636
|
+
const filteredEntries = options?.bypassRecentDedupe === true
|
|
1637
|
+
? new Map(entries)
|
|
1638
|
+
: new Map();
|
|
1639
|
+
if (options?.bypassRecentDedupe !== true) {
|
|
1640
|
+
for (const [hash, entry] of entries) {
|
|
1641
|
+
const prev = recentlyDispatchedByHash.get(hash);
|
|
1642
|
+
if (prev != null && now - prev <= RECENT_REPAIR_DISPATCH_TTL_MS) {
|
|
1643
|
+
continue;
|
|
1644
|
+
}
|
|
1645
|
+
recentlyDispatchedByHash.set(hash, now);
|
|
1646
|
+
filteredEntries.set(hash, entry);
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
else {
|
|
1650
|
+
for (const hash of entries.keys()) {
|
|
1651
|
+
recentlyDispatchedByHash.set(hash, now);
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1654
|
+
if (filteredEntries.size === 0) {
|
|
1655
|
+
return;
|
|
1656
|
+
}
|
|
1657
|
+
const run = () => {
|
|
1658
|
+
// For force-fresh churn repair we intentionally bypass rateless IBLT and
|
|
1659
|
+
// use simple hash-based sync. This path is a directed "push these hashes
|
|
1660
|
+
// to that peer" recovery flow; using simple sync here avoids occasional
|
|
1661
|
+
// single-hash gaps seen with IBLT-oriented maybe-sync batches under churn.
|
|
1662
|
+
if (options?.forceFreshDelivery &&
|
|
1663
|
+
this.syncronizer instanceof RatelessIBLTSynchronizer) {
|
|
1664
|
+
return Promise.resolve(this.syncronizer.simple.onMaybeMissingEntries({
|
|
1665
|
+
entries: filteredEntries,
|
|
1666
|
+
targets: [target],
|
|
1667
|
+
})).catch((error) => logger.error(error));
|
|
1668
|
+
}
|
|
1669
|
+
return Promise.resolve(this.syncronizer.onMaybeMissingEntries({
|
|
1670
|
+
entries: filteredEntries,
|
|
1671
|
+
targets: [target],
|
|
1672
|
+
})).catch((error) => logger.error(error));
|
|
1673
|
+
};
|
|
1674
|
+
const retrySchedule = options?.retryScheduleMs && options.retryScheduleMs.length > 0
|
|
1675
|
+
? options.retryScheduleMs
|
|
1676
|
+
: options?.forceFreshDelivery
|
|
1677
|
+
? FORCE_FRESH_RETRY_SCHEDULE_MS
|
|
1678
|
+
: [0];
|
|
1679
|
+
for (const delayMs of retrySchedule) {
|
|
1680
|
+
if (delayMs === 0) {
|
|
1681
|
+
void run();
|
|
1682
|
+
continue;
|
|
1683
|
+
}
|
|
1684
|
+
const timer = setTimeout(() => {
|
|
1685
|
+
this._repairRetryTimers.delete(timer);
|
|
1686
|
+
if (this.closed) {
|
|
1687
|
+
return;
|
|
1688
|
+
}
|
|
1689
|
+
void run();
|
|
1690
|
+
}, delayMs);
|
|
1691
|
+
timer.unref?.();
|
|
1692
|
+
this._repairRetryTimers.add(timer);
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
scheduleRepairSweep(options) {
|
|
1696
|
+
if (options.forceFreshDelivery) {
|
|
1697
|
+
this._repairSweepForceFreshPending = true;
|
|
1698
|
+
}
|
|
1699
|
+
for (const peer of options.addedPeers) {
|
|
1700
|
+
this._repairSweepAddedPeersPending.add(peer);
|
|
1701
|
+
}
|
|
1702
|
+
if (!this._repairSweepRunning && !this.closed) {
|
|
1703
|
+
this._repairSweepRunning = true;
|
|
1704
|
+
void this.runRepairSweep();
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
async runRepairSweep() {
|
|
1708
|
+
try {
|
|
1709
|
+
while (!this.closed) {
|
|
1710
|
+
const forceFreshDelivery = this._repairSweepForceFreshPending;
|
|
1711
|
+
const addedPeers = new Set(this._repairSweepAddedPeersPending);
|
|
1712
|
+
this._repairSweepForceFreshPending = false;
|
|
1713
|
+
this._repairSweepAddedPeersPending.clear();
|
|
1714
|
+
if (!forceFreshDelivery && addedPeers.size === 0) {
|
|
1715
|
+
return;
|
|
1716
|
+
}
|
|
1717
|
+
const pendingByTarget = new Map();
|
|
1718
|
+
const flushTarget = (target) => {
|
|
1719
|
+
const entries = pendingByTarget.get(target);
|
|
1720
|
+
if (!entries || entries.size === 0) {
|
|
1721
|
+
return;
|
|
1722
|
+
}
|
|
1723
|
+
const isJoinWarmupTarget = addedPeers.has(target);
|
|
1724
|
+
const bypassRecentDedupe = isJoinWarmupTarget || forceFreshDelivery;
|
|
1725
|
+
this.dispatchMaybeMissingEntries(target, entries, {
|
|
1726
|
+
bypassRecentDedupe,
|
|
1727
|
+
retryScheduleMs: isJoinWarmupTarget
|
|
1728
|
+
? JOIN_WARMUP_RETRY_SCHEDULE_MS
|
|
1729
|
+
: undefined,
|
|
1730
|
+
forceFreshDelivery,
|
|
1731
|
+
});
|
|
1732
|
+
pendingByTarget.delete(target);
|
|
1733
|
+
};
|
|
1734
|
+
const queueEntryForTarget = (target, entry) => {
|
|
1735
|
+
let set = pendingByTarget.get(target);
|
|
1736
|
+
if (!set) {
|
|
1737
|
+
set = new Map();
|
|
1738
|
+
pendingByTarget.set(target, set);
|
|
1739
|
+
}
|
|
1740
|
+
if (set.has(entry.hash)) {
|
|
1741
|
+
return;
|
|
1742
|
+
}
|
|
1743
|
+
set.set(entry.hash, entry);
|
|
1744
|
+
if (set.size >= this.repairSweepTargetBufferSize) {
|
|
1745
|
+
flushTarget(target);
|
|
1746
|
+
}
|
|
1747
|
+
};
|
|
1748
|
+
const iterator = this.entryCoordinatesIndex.iterate({});
|
|
1749
|
+
try {
|
|
1750
|
+
while (!this.closed && !iterator.done()) {
|
|
1751
|
+
const entries = await iterator.next(REPAIR_SWEEP_ENTRY_BATCH_SIZE);
|
|
1752
|
+
for (const entry of entries) {
|
|
1753
|
+
const entryReplicated = entry.value;
|
|
1754
|
+
const currentPeers = await this.findLeaders(entryReplicated.coordinates, entryReplicated, { roleAge: 0 });
|
|
1755
|
+
if (forceFreshDelivery) {
|
|
1756
|
+
for (const [currentPeer] of currentPeers) {
|
|
1757
|
+
if (currentPeer === this.node.identity.publicKey.hashcode()) {
|
|
1758
|
+
continue;
|
|
1759
|
+
}
|
|
1760
|
+
queueEntryForTarget(currentPeer, entryReplicated);
|
|
1761
|
+
}
|
|
1762
|
+
}
|
|
1763
|
+
if (addedPeers.size > 0) {
|
|
1764
|
+
for (const peer of addedPeers) {
|
|
1765
|
+
if (currentPeers.has(peer)) {
|
|
1766
|
+
queueEntryForTarget(peer, entryReplicated);
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
}
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
finally {
|
|
1774
|
+
await iterator.close();
|
|
1775
|
+
}
|
|
1776
|
+
for (const target of [...pendingByTarget.keys()]) {
|
|
1777
|
+
flushTarget(target);
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
}
|
|
1781
|
+
catch (error) {
|
|
1782
|
+
if (!isNotStartedError(error)) {
|
|
1783
|
+
logger.error(`Repair sweep failed: ${error?.message ?? error}`);
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
finally {
|
|
1787
|
+
this._repairSweepRunning = false;
|
|
1788
|
+
if (!this.closed &&
|
|
1789
|
+
(this._repairSweepForceFreshPending ||
|
|
1790
|
+
this._repairSweepAddedPeersPending.size > 0)) {
|
|
1791
|
+
this._repairSweepRunning = true;
|
|
1792
|
+
void this.runRepairSweep();
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1525
1796
|
async pruneDebouncedFnAddIfNotKeeping(args) {
|
|
1526
1797
|
if (!this.keep || !(await this.keep(args.value.entry))) {
|
|
1527
1798
|
return this.pruneDebouncedFn.add(args);
|
|
@@ -1693,6 +1964,11 @@ let SharedLog = (() => {
|
|
|
1693
1964
|
this._replicationInfoBlockedPeers = new Set();
|
|
1694
1965
|
this._replicationInfoRequestByPeer = new Map();
|
|
1695
1966
|
this._replicationInfoApplyQueueByPeer = new Map();
|
|
1967
|
+
this._repairRetryTimers = new Set();
|
|
1968
|
+
this._recentRepairDispatch = new Map();
|
|
1969
|
+
this._repairSweepRunning = false;
|
|
1970
|
+
this._repairSweepForceFreshPending = false;
|
|
1971
|
+
this._repairSweepAddedPeersPending = new Set();
|
|
1696
1972
|
this.coordinateToHash = new Cache({ max: 1e6, ttl: 1e4 });
|
|
1697
1973
|
this.recentlyRebalanced = new Cache({ max: 1e4, ttl: 1e5 });
|
|
1698
1974
|
this.uniqueReplicators = new Set();
|
|
@@ -1702,6 +1978,7 @@ let SharedLog = (() => {
|
|
|
1702
1978
|
this.oldestOpenTime = this.openTime;
|
|
1703
1979
|
this.distributionDebounceTime =
|
|
1704
1980
|
options?.distributionDebounceTime || DEFAULT_DISTRIBUTION_DEBOUNCE_TIME; // expect > 0
|
|
1981
|
+
this.repairSweepTargetBufferSize = toPositiveInteger(options?.sync?.repairSweepTargetBufferSize, REPAIR_SWEEP_TARGET_BUFFER_SIZE, "sync.repairSweepTargetBufferSize");
|
|
1705
1982
|
this.timeUntilRoleMaturity =
|
|
1706
1983
|
options?.timeUntilRoleMaturity ?? WAIT_FOR_ROLE_MATURITY;
|
|
1707
1984
|
this.waitForReplicatorTimeout =
|
|
@@ -2298,6 +2575,14 @@ let SharedLog = (() => {
|
|
|
2298
2575
|
clearInterval(this.interval);
|
|
2299
2576
|
this.node.services.pubsub.removeEventListener("subscribe", this._onSubscriptionFn);
|
|
2300
2577
|
this.node.services.pubsub.removeEventListener("unsubscribe", this._onUnsubscriptionFn);
|
|
2578
|
+
for (const timer of this._repairRetryTimers) {
|
|
2579
|
+
clearTimeout(timer);
|
|
2580
|
+
}
|
|
2581
|
+
this._repairRetryTimers.clear();
|
|
2582
|
+
this._recentRepairDispatch.clear();
|
|
2583
|
+
this._repairSweepRunning = false;
|
|
2584
|
+
this._repairSweepForceFreshPending = false;
|
|
2585
|
+
this._repairSweepAddedPeersPending.clear();
|
|
2301
2586
|
for (const [_k, v] of this._pendingDeletes) {
|
|
2302
2587
|
v.clear();
|
|
2303
2588
|
v.promise.resolve(); // TODO or reject?
|
|
@@ -3219,19 +3504,33 @@ let SharedLog = (() => {
|
|
|
3219
3504
|
async _waitForReplicators(cursors, entry, waitFor, options = { timeout: this.waitForReplicatorTimeout }) {
|
|
3220
3505
|
const timeout = options.timeout ?? this.waitForReplicatorTimeout;
|
|
3221
3506
|
return new Promise((resolve, reject) => {
|
|
3507
|
+
let settled = false;
|
|
3222
3508
|
const removeListeners = () => {
|
|
3223
3509
|
this.events.removeEventListener("replication:change", roleListener);
|
|
3224
3510
|
this.events.removeEventListener("replicator:mature", roleListener); // TODO replication:change event ?
|
|
3225
3511
|
this._closeController.signal.removeEventListener("abort", abortListener);
|
|
3226
3512
|
};
|
|
3227
|
-
const
|
|
3513
|
+
const settleResolve = (value) => {
|
|
3514
|
+
if (settled)
|
|
3515
|
+
return;
|
|
3516
|
+
settled = true;
|
|
3228
3517
|
removeListeners();
|
|
3229
3518
|
clearTimeout(timer);
|
|
3230
|
-
resolve(
|
|
3519
|
+
resolve(value);
|
|
3231
3520
|
};
|
|
3232
|
-
const
|
|
3521
|
+
const settleReject = (error) => {
|
|
3522
|
+
if (settled)
|
|
3523
|
+
return;
|
|
3524
|
+
settled = true;
|
|
3233
3525
|
removeListeners();
|
|
3234
|
-
|
|
3526
|
+
clearTimeout(timer);
|
|
3527
|
+
reject(error);
|
|
3528
|
+
};
|
|
3529
|
+
const abortListener = () => {
|
|
3530
|
+
settleResolve(false);
|
|
3531
|
+
};
|
|
3532
|
+
const timer = setTimeout(async () => {
|
|
3533
|
+
settleResolve(false);
|
|
3235
3534
|
}, timeout);
|
|
3236
3535
|
const check = async () => {
|
|
3237
3536
|
let leaderKeys = new Set();
|
|
@@ -3251,17 +3550,20 @@ let SharedLog = (() => {
|
|
|
3251
3550
|
}
|
|
3252
3551
|
}
|
|
3253
3552
|
options?.onLeader && leaderKeys.forEach(options.onLeader);
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3553
|
+
settleResolve(leaders);
|
|
3554
|
+
};
|
|
3555
|
+
const runCheck = () => {
|
|
3556
|
+
void check().catch((error) => {
|
|
3557
|
+
settleReject(error);
|
|
3558
|
+
});
|
|
3257
3559
|
};
|
|
3258
3560
|
const roleListener = () => {
|
|
3259
|
-
|
|
3561
|
+
runCheck();
|
|
3260
3562
|
};
|
|
3261
3563
|
this.events.addEventListener("replication:change", roleListener); // TODO replication:change event ?
|
|
3262
3564
|
this.events.addEventListener("replicator:mature", roleListener); // TODO replication:change event ?
|
|
3263
3565
|
this._closeController.signal.addEventListener("abort", abortListener);
|
|
3264
|
-
|
|
3566
|
+
runCheck();
|
|
3265
3567
|
});
|
|
3266
3568
|
}
|
|
3267
3569
|
async createCoordinates(entry, minReplicas) {
|
|
@@ -3386,8 +3688,8 @@ let SharedLog = (() => {
|
|
|
3386
3688
|
const roleAge = options?.roleAge ?? (await this.getDefaultMinRoleAge()); // TODO -500 as is added so that i f someone else is just as new as us, then we treat them as mature as us. without -500 we might be slower syncing if two nodes starts almost at the same time
|
|
3387
3689
|
const selfHash = this.node.identity.publicKey.hashcode();
|
|
3388
3690
|
// Prefer `uniqueReplicators` (replicator cache) as soon as it has any data.
|
|
3389
|
-
//
|
|
3390
|
-
//
|
|
3691
|
+
// If it is still warming up (for example, only contains self), supplement with
|
|
3692
|
+
// current subscribers until we have enough candidates for this decision.
|
|
3391
3693
|
let peerFilter = undefined;
|
|
3392
3694
|
const selfReplicating = await this.isReplicating();
|
|
3393
3695
|
if (this.uniqueReplicators.size > 0) {
|
|
@@ -3398,6 +3700,23 @@ let SharedLog = (() => {
|
|
|
3398
3700
|
else {
|
|
3399
3701
|
peerFilter.delete(selfHash);
|
|
3400
3702
|
}
|
|
3703
|
+
try {
|
|
3704
|
+
const subscribers = await this._getTopicSubscribers(this.topic);
|
|
3705
|
+
if (subscribers && subscribers.length > 0) {
|
|
3706
|
+
for (const subscriber of subscribers) {
|
|
3707
|
+
peerFilter.add(subscriber.hashcode());
|
|
3708
|
+
}
|
|
3709
|
+
if (selfReplicating) {
|
|
3710
|
+
peerFilter.add(selfHash);
|
|
3711
|
+
}
|
|
3712
|
+
else {
|
|
3713
|
+
peerFilter.delete(selfHash);
|
|
3714
|
+
}
|
|
3715
|
+
}
|
|
3716
|
+
}
|
|
3717
|
+
catch {
|
|
3718
|
+
// Best-effort only; keep current peerFilter.
|
|
3719
|
+
}
|
|
3401
3720
|
}
|
|
3402
3721
|
else {
|
|
3403
3722
|
try {
|
|
@@ -3505,9 +3824,20 @@ let SharedLog = (() => {
|
|
|
3505
3824
|
this._replicationInfoBlockedPeers.add(peerHash);
|
|
3506
3825
|
}
|
|
3507
3826
|
if (!subscribed) {
|
|
3827
|
+
const wasReplicator = this.uniqueReplicators.has(peerHash);
|
|
3828
|
+
try {
|
|
3829
|
+
// Unsubscribe can race with the peer's final replication reset message.
|
|
3830
|
+
// Proactively evict its ranges so leader selection doesn't keep stale owners.
|
|
3831
|
+
await this.removeReplicator(publicKey, { noEvent: true });
|
|
3832
|
+
}
|
|
3833
|
+
catch (error) {
|
|
3834
|
+
if (!isNotStartedError(error)) {
|
|
3835
|
+
throw error;
|
|
3836
|
+
}
|
|
3837
|
+
}
|
|
3508
3838
|
// Emit replicator:leave at most once per (join -> leave) transition, even if we
|
|
3509
3839
|
// concurrently process unsubscribe + replication reset messages for the same peer.
|
|
3510
|
-
const stoppedTransition =
|
|
3840
|
+
const stoppedTransition = wasReplicator;
|
|
3511
3841
|
this._replicatorJoinEmitted.delete(peerHash);
|
|
3512
3842
|
this.cancelReplicationInfoRequests(peerHash);
|
|
3513
3843
|
this.removePeerFromGidPeerHistory(peerHash);
|
|
@@ -3876,14 +4206,79 @@ let SharedLog = (() => {
|
|
|
3876
4206
|
? changeOrChanges
|
|
3877
4207
|
: [changeOrChanges];
|
|
3878
4208
|
const changes = batchedChanges.flat();
|
|
4209
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
3879
4210
|
// On removed ranges (peer leaves / shrink), gid-level history can hide
|
|
3880
4211
|
// per-entry gaps. Force a fresh delivery pass for reassigned entries.
|
|
3881
|
-
const forceFreshDelivery = changes.some((change) => change.type === "removed");
|
|
4212
|
+
const forceFreshDelivery = changes.some((change) => change.type === "removed" && change.range.hash !== selfHash);
|
|
3882
4213
|
const gidPeersHistorySnapshot = new Map();
|
|
4214
|
+
const dedupeCutoff = Date.now() - RECENT_REPAIR_DISPATCH_TTL_MS;
|
|
4215
|
+
for (const [target, hashes] of this._recentRepairDispatch) {
|
|
4216
|
+
for (const [hash, ts] of hashes) {
|
|
4217
|
+
if (ts <= dedupeCutoff) {
|
|
4218
|
+
hashes.delete(hash);
|
|
4219
|
+
}
|
|
4220
|
+
}
|
|
4221
|
+
if (hashes.size === 0) {
|
|
4222
|
+
this._recentRepairDispatch.delete(target);
|
|
4223
|
+
}
|
|
4224
|
+
}
|
|
3883
4225
|
const changed = false;
|
|
4226
|
+
const replacedPeers = new Set();
|
|
4227
|
+
for (const change of changes) {
|
|
4228
|
+
if (change.type === "replaced" && change.range.hash !== selfHash) {
|
|
4229
|
+
replacedPeers.add(change.range.hash);
|
|
4230
|
+
}
|
|
4231
|
+
}
|
|
4232
|
+
const addedPeers = new Set();
|
|
4233
|
+
for (const change of changes) {
|
|
4234
|
+
if (change.type === "added" || change.type === "replaced") {
|
|
4235
|
+
const hash = change.range.hash;
|
|
4236
|
+
if (hash !== selfHash) {
|
|
4237
|
+
// Range updates can reassign entries to an existing peer shortly after it
|
|
4238
|
+
// already received a subset. Avoid suppressing legitimate follow-up repair.
|
|
4239
|
+
this._recentRepairDispatch.delete(hash);
|
|
4240
|
+
}
|
|
4241
|
+
}
|
|
4242
|
+
if (change.type === "added") {
|
|
4243
|
+
const hash = change.range.hash;
|
|
4244
|
+
if (hash !== selfHash && !replacedPeers.has(hash)) {
|
|
4245
|
+
addedPeers.add(hash);
|
|
4246
|
+
}
|
|
4247
|
+
}
|
|
4248
|
+
}
|
|
3884
4249
|
try {
|
|
3885
4250
|
const uncheckedDeliver = new Map();
|
|
3886
|
-
|
|
4251
|
+
const flushUncheckedDeliverTarget = (target) => {
|
|
4252
|
+
const entries = uncheckedDeliver.get(target);
|
|
4253
|
+
if (!entries || entries.size === 0) {
|
|
4254
|
+
return;
|
|
4255
|
+
}
|
|
4256
|
+
const isJoinWarmupTarget = addedPeers.has(target);
|
|
4257
|
+
const bypassRecentDedupe = isJoinWarmupTarget || forceFreshDelivery;
|
|
4258
|
+
this.dispatchMaybeMissingEntries(target, entries, {
|
|
4259
|
+
bypassRecentDedupe,
|
|
4260
|
+
retryScheduleMs: isJoinWarmupTarget
|
|
4261
|
+
? JOIN_WARMUP_RETRY_SCHEDULE_MS
|
|
4262
|
+
: undefined,
|
|
4263
|
+
forceFreshDelivery,
|
|
4264
|
+
});
|
|
4265
|
+
uncheckedDeliver.delete(target);
|
|
4266
|
+
};
|
|
4267
|
+
const queueUncheckedDeliver = (target, entry) => {
|
|
4268
|
+
let set = uncheckedDeliver.get(target);
|
|
4269
|
+
if (!set) {
|
|
4270
|
+
set = new Map();
|
|
4271
|
+
uncheckedDeliver.set(target, set);
|
|
4272
|
+
}
|
|
4273
|
+
if (set.has(entry.hash)) {
|
|
4274
|
+
return;
|
|
4275
|
+
}
|
|
4276
|
+
set.set(entry.hash, entry);
|
|
4277
|
+
if (set.size >= this.repairSweepTargetBufferSize) {
|
|
4278
|
+
flushUncheckedDeliverTarget(target);
|
|
4279
|
+
}
|
|
4280
|
+
};
|
|
4281
|
+
for await (const entryReplicated of toRebalance(changes, this.entryCoordinatesIndex, this.recentlyRebalanced, { forceFresh: forceFreshDelivery })) {
|
|
3887
4282
|
if (this.closed) {
|
|
3888
4283
|
break;
|
|
3889
4284
|
}
|
|
@@ -3909,14 +4304,7 @@ let SharedLog = (() => {
|
|
|
3909
4304
|
continue;
|
|
3910
4305
|
}
|
|
3911
4306
|
if (!oldPeersSet?.has(currentPeer)) {
|
|
3912
|
-
|
|
3913
|
-
if (!set) {
|
|
3914
|
-
set = new Map();
|
|
3915
|
-
uncheckedDeliver.set(currentPeer, set);
|
|
3916
|
-
}
|
|
3917
|
-
if (!set.has(entryReplicated.hash)) {
|
|
3918
|
-
set.set(entryReplicated.hash, entryReplicated);
|
|
3919
|
-
}
|
|
4307
|
+
queueUncheckedDeliver(currentPeer, entryReplicated);
|
|
3920
4308
|
}
|
|
3921
4309
|
}
|
|
3922
4310
|
if (oldPeersSet) {
|
|
@@ -3942,11 +4330,13 @@ let SharedLog = (() => {
|
|
|
3942
4330
|
this.removePruneRequestSent(entryReplicated.hash);
|
|
3943
4331
|
}
|
|
3944
4332
|
}
|
|
3945
|
-
|
|
3946
|
-
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
4333
|
+
if (forceFreshDelivery || addedPeers.size > 0) {
|
|
4334
|
+
// Schedule a coalesced background sweep for churn/join windows instead of
|
|
4335
|
+
// scanning the whole index synchronously on each replication change.
|
|
4336
|
+
this.scheduleRepairSweep({ forceFreshDelivery, addedPeers });
|
|
4337
|
+
}
|
|
4338
|
+
for (const target of [...uncheckedDeliver.keys()]) {
|
|
4339
|
+
flushUncheckedDeliverTarget(target);
|
|
3950
4340
|
}
|
|
3951
4341
|
return changed;
|
|
3952
4342
|
}
|
|
@@ -3965,6 +4355,7 @@ let SharedLog = (() => {
|
|
|
3965
4355
|
}
|
|
3966
4356
|
const fromHash = evt.detail.from.hashcode();
|
|
3967
4357
|
this._replicationInfoBlockedPeers.add(fromHash);
|
|
4358
|
+
this._recentRepairDispatch.delete(fromHash);
|
|
3968
4359
|
// Keep a per-peer timestamp watermark when we observe an unsubscribe. This
|
|
3969
4360
|
// prevents late/out-of-order replication-info messages from re-introducing
|
|
3970
4361
|
// stale segments for a peer that has already left the topic.
|
|
@@ -3982,7 +4373,7 @@ let SharedLog = (() => {
|
|
|
3982
4373
|
}
|
|
3983
4374
|
this.remoteBlocks.onReachable(evt.detail.from);
|
|
3984
4375
|
this._replicationInfoBlockedPeers.delete(evt.detail.from.hashcode());
|
|
3985
|
-
|
|
4376
|
+
await this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, true);
|
|
3986
4377
|
}
|
|
3987
4378
|
async rebalanceParticipation() {
|
|
3988
4379
|
// update more participation rate to converge to the average expected rate or bounded by
|