@peerbit/shared-log 16.0.29 → 16.0.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +306 -42
- package/dist/src/index.js.map +1 -1
- package/dist/src/sync/index.d.ts +7 -0
- package/dist/src/sync/index.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.d.ts +1 -0
- package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.js +8 -3
- package/dist/src/sync/rateless-iblt.js.map +1 -1
- package/dist/src/sync/simple.d.ts +1 -0
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +16 -3
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +15 -15
- package/src/index.ts +330 -56
- package/src/sync/index.ts +7 -0
- package/src/sync/rateless-iblt.ts +9 -3
- package/src/sync/simple.ts +17 -3
package/src/index.ts
CHANGED
|
@@ -564,7 +564,7 @@ const emitAdvisorySyncProfileDuration = (
|
|
|
564
564
|
try {
|
|
565
565
|
emitSyncProfileDuration(profile, startedAt, event);
|
|
566
566
|
} catch {
|
|
567
|
-
//
|
|
567
|
+
// Advisory diagnostics must not change replication or lifecycle behavior.
|
|
568
568
|
}
|
|
569
569
|
};
|
|
570
570
|
|
|
@@ -4033,7 +4033,13 @@ export class SharedLog<
|
|
|
4033
4033
|
this._repairMetrics["join-warmup"].simpleFallbackPasses += 1;
|
|
4034
4034
|
},
|
|
4035
4035
|
sendEntriesSimple: (target, entries, options) =>
|
|
4036
|
-
this.sendRepairEntriesWithTransport(
|
|
4036
|
+
this.sendRepairEntriesWithTransport(
|
|
4037
|
+
target,
|
|
4038
|
+
entries,
|
|
4039
|
+
"simple",
|
|
4040
|
+
options,
|
|
4041
|
+
"join-warmup",
|
|
4042
|
+
),
|
|
4037
4043
|
logError: (error) => logger.error(error),
|
|
4038
4044
|
});
|
|
4039
4045
|
}
|
|
@@ -6346,6 +6352,95 @@ export class SharedLog<
|
|
|
6346
6352
|
records.size,
|
|
6347
6353
|
);
|
|
6348
6354
|
const signal = deadline.signal;
|
|
6355
|
+
const recoveryController = new AbortController();
|
|
6356
|
+
const recoverySignal = AbortSignal.any([signal, recoveryController.signal]);
|
|
6357
|
+
const recoveryByPeer = new Map<
|
|
6358
|
+
string,
|
|
6359
|
+
{
|
|
6360
|
+
controller: AbortController;
|
|
6361
|
+
timer: ReturnType<typeof setTimeout>;
|
|
6362
|
+
}
|
|
6363
|
+
>();
|
|
6364
|
+
let recoveryCursor = 0;
|
|
6365
|
+
const recoverSelectedPeers = (selected: Set<string>) => {
|
|
6366
|
+
// Recovery is advisory work, not a receipt or a replacement leader plan.
|
|
6367
|
+
// Never occupy transfer/request slots while waiting for it. The separate
|
|
6368
|
+
// bounded pool rotates through fresh candidates so quiet/incomplete peers
|
|
6369
|
+
// cannot indefinitely hide a later recoverable peer.
|
|
6370
|
+
for (const [peer, state] of recoveryByPeer) {
|
|
6371
|
+
if (!selected.has(peer)) {
|
|
6372
|
+
clearTimeout(state.timer);
|
|
6373
|
+
state.controller.abort();
|
|
6374
|
+
}
|
|
6375
|
+
}
|
|
6376
|
+
const peers = [...selected];
|
|
6377
|
+
for (
|
|
6378
|
+
let visited = 0;
|
|
6379
|
+
visited < peers.length &&
|
|
6380
|
+
recoveryByPeer.size < MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL &&
|
|
6381
|
+
!recoverySignal.aborted;
|
|
6382
|
+
visited++
|
|
6383
|
+
) {
|
|
6384
|
+
const peer = peers[recoveryCursor++ % peers.length]!;
|
|
6385
|
+
if (recoveryByPeer.has(peer)) continue;
|
|
6386
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
6387
|
+
if (
|
|
6388
|
+
current &&
|
|
6389
|
+
this._v2Send.isLatestConfirmedForPeer({
|
|
6390
|
+
peerHash: peer,
|
|
6391
|
+
peerSession: current.peerSession,
|
|
6392
|
+
receiverTransportSession: current.capabilitySession,
|
|
6393
|
+
})
|
|
6394
|
+
) {
|
|
6395
|
+
continue;
|
|
6396
|
+
}
|
|
6397
|
+
const expiresAt = Math.min(
|
|
6398
|
+
deadline.deadline,
|
|
6399
|
+
Date.now() + MAX_PERSISTED_RECEIPT_ATTEMPT_MS,
|
|
6400
|
+
);
|
|
6401
|
+
const controller = new AbortController();
|
|
6402
|
+
const attemptSignal = AbortSignal.any([
|
|
6403
|
+
recoverySignal,
|
|
6404
|
+
controller.signal,
|
|
6405
|
+
]);
|
|
6406
|
+
const timer = setTimeout(
|
|
6407
|
+
() => controller.abort(),
|
|
6408
|
+
Math.max(1, expiresAt - Date.now()),
|
|
6409
|
+
);
|
|
6410
|
+
timer.unref?.();
|
|
6411
|
+
const state = { controller, timer };
|
|
6412
|
+
recoveryByPeer.set(peer, state);
|
|
6413
|
+
void (async () => {
|
|
6414
|
+
// Resolve only an authenticated transport-cache key. Keep this slot
|
|
6415
|
+
// reserved until the lookup actually settles, even if a custom
|
|
6416
|
+
// resolver ignores cancellation, rather than launching duplicates.
|
|
6417
|
+
const key = await this._resolvePublicKeyFromHash(peer);
|
|
6418
|
+
if (
|
|
6419
|
+
attemptSignal.aborted ||
|
|
6420
|
+
!key ||
|
|
6421
|
+
key.hashcode() !== peer ||
|
|
6422
|
+
Date.now() >= expiresAt
|
|
6423
|
+
) {
|
|
6424
|
+
return;
|
|
6425
|
+
}
|
|
6426
|
+
// Reuse the exact-session watchdog, capability/subscriber recovery,
|
|
6427
|
+
// and replacement-session handling from the public preflight. Its
|
|
6428
|
+
// result is ignored: settlement still checks the exact entry leaders
|
|
6429
|
+
// and accepts only the subsequent durable, session-bound receipts.
|
|
6430
|
+
await this.waitForPersistedReceiptPeerReadiness(key, {
|
|
6431
|
+
timeout: Math.max(1, expiresAt - Date.now()),
|
|
6432
|
+
signal: attemptSignal,
|
|
6433
|
+
});
|
|
6434
|
+
})()
|
|
6435
|
+
.catch(() => undefined)
|
|
6436
|
+
.finally(() => {
|
|
6437
|
+
clearTimeout(timer);
|
|
6438
|
+
if (recoveryByPeer.get(peer) === state) {
|
|
6439
|
+
recoveryByPeer.delete(peer);
|
|
6440
|
+
}
|
|
6441
|
+
});
|
|
6442
|
+
}
|
|
6443
|
+
};
|
|
6349
6444
|
let maxAttemptMs = MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
6350
6445
|
let initialTransferPending = transferOnFirstRound;
|
|
6351
6446
|
let needsInitialLeaderCheck = true;
|
|
@@ -6412,6 +6507,7 @@ export class SharedLog<
|
|
|
6412
6507
|
// transport epoch. A revision/session change purges them before they can
|
|
6413
6508
|
// survive an away-and-back leader transition or combine with a later peer.
|
|
6414
6509
|
const hashesByPeer = new Map<string, string[]>();
|
|
6510
|
+
const recoveryCandidates = new Set<string>();
|
|
6415
6511
|
const entryArray = [...records.values()];
|
|
6416
6512
|
const leadersByEntry = await this.planPersistedDeliveryLeaders(
|
|
6417
6513
|
entryArray,
|
|
@@ -6456,6 +6552,7 @@ export class SharedLog<
|
|
|
6456
6552
|
if (acknowledgements.size >= minAcks) continue;
|
|
6457
6553
|
for (const peer of leaders.keys()) {
|
|
6458
6554
|
if (peer === selfHash) continue;
|
|
6555
|
+
recoveryCandidates.add(peer);
|
|
6459
6556
|
const current = this.persistedReceiptPeerSession(peer);
|
|
6460
6557
|
if (!current) continue;
|
|
6461
6558
|
if (acknowledgements.has(peer)) continue;
|
|
@@ -6465,6 +6562,7 @@ export class SharedLog<
|
|
|
6465
6562
|
}
|
|
6466
6563
|
}
|
|
6467
6564
|
if (!isRoundOwnershipCurrent()) continue;
|
|
6565
|
+
recoverSelectedPeers(recoveryCandidates);
|
|
6468
6566
|
|
|
6469
6567
|
const operationQueue = new PQueue({
|
|
6470
6568
|
concurrency: MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL,
|
|
@@ -6825,6 +6923,11 @@ export class SharedLog<
|
|
|
6825
6923
|
}
|
|
6826
6924
|
throw new PersistedDeliveryError(error, committedHashes);
|
|
6827
6925
|
} finally {
|
|
6926
|
+
recoveryController.abort();
|
|
6927
|
+
for (const state of recoveryByPeer.values()) {
|
|
6928
|
+
clearTimeout(state.timer);
|
|
6929
|
+
state.controller.abort();
|
|
6930
|
+
}
|
|
6828
6931
|
if (ownedDeadline) deadline.dispose();
|
|
6829
6932
|
}
|
|
6830
6933
|
}
|
|
@@ -10639,6 +10742,7 @@ export class SharedLog<
|
|
|
10639
10742
|
isStillCurrent?: () => boolean;
|
|
10640
10743
|
signal?: AbortSignal;
|
|
10641
10744
|
},
|
|
10745
|
+
mode?: RepairDispatchMode,
|
|
10642
10746
|
) {
|
|
10643
10747
|
const isStillCurrent = options?.isStillCurrent ?? (() => true);
|
|
10644
10748
|
if (!isStillCurrent()) {
|
|
@@ -10646,53 +10750,87 @@ export class SharedLog<
|
|
|
10646
10750
|
}
|
|
10647
10751
|
const unknownEntries = new Map<string, RepairDispatchEntry<R>>();
|
|
10648
10752
|
const knownHashes: string[] = [];
|
|
10649
|
-
|
|
10650
|
-
|
|
10651
|
-
|
|
10652
|
-
|
|
10653
|
-
|
|
10654
|
-
|
|
10655
|
-
|
|
10656
|
-
|
|
10657
|
-
|
|
10658
|
-
|
|
10659
|
-
|
|
10660
|
-
|
|
10661
|
-
|
|
10753
|
+
const profile = this._logProperties?.sync?.profile;
|
|
10754
|
+
const startedAt = syncProfileStart(profile);
|
|
10755
|
+
const inputEntries = profile ? entries.size : 0;
|
|
10756
|
+
let selectedEntries = 0;
|
|
10757
|
+
let lastObservedCurrent = true;
|
|
10758
|
+
let outcome = "stale";
|
|
10759
|
+
try {
|
|
10760
|
+
for (const [hash, entry] of entries) {
|
|
10761
|
+
if (
|
|
10762
|
+
(options?.bypassRecentKnownPeers ||
|
|
10763
|
+
!this.isEntryRecentlyKnownByPeer(
|
|
10764
|
+
hash,
|
|
10765
|
+
target,
|
|
10766
|
+
RECENT_KNOWN_REPAIR_SUPPRESSION_MS,
|
|
10767
|
+
)) &&
|
|
10768
|
+
(options?.bypassKnownPeers || !this.isEntryKnownByPeer(hash, target))
|
|
10769
|
+
) {
|
|
10770
|
+
unknownEntries.set(hash, entry);
|
|
10771
|
+
} else {
|
|
10772
|
+
knownHashes.push(hash);
|
|
10773
|
+
}
|
|
10662
10774
|
}
|
|
10663
|
-
|
|
10664
|
-
|
|
10665
|
-
return;
|
|
10666
|
-
|
|
10667
|
-
|
|
10668
|
-
|
|
10669
|
-
|
|
10670
|
-
|
|
10671
|
-
|
|
10672
|
-
|
|
10673
|
-
|
|
10674
|
-
|
|
10675
|
-
|
|
10676
|
-
|
|
10677
|
-
|
|
10678
|
-
|
|
10679
|
-
|
|
10680
|
-
return;
|
|
10681
|
-
}
|
|
10682
|
-
|
|
10683
|
-
const syncEntries = this._logProperties?.sync?.priority
|
|
10684
|
-
? (this._coordinates.materializeRepairDispatchEntries(
|
|
10775
|
+
// A custom synchronizer may mutate the Map once it receives it.
|
|
10776
|
+
if (profile) selectedEntries = unknownEntries.size;
|
|
10777
|
+
if (!isStillCurrent()) return;
|
|
10778
|
+
this.clearRepairFrontierHashes(target, knownHashes);
|
|
10779
|
+
if (unknownEntries.size === 0) {
|
|
10780
|
+
outcome = "known-suppressed";
|
|
10781
|
+
return;
|
|
10782
|
+
}
|
|
10783
|
+
if (transport === "simple") {
|
|
10784
|
+
// Observe only checks the lower path already makes, without adding
|
|
10785
|
+
// lifecycle decisions or wrapping the disabled-profiling path.
|
|
10786
|
+
const dispatchIsStillCurrent = profile
|
|
10787
|
+
? () => (lastObservedCurrent = isStillCurrent())
|
|
10788
|
+
: isStillCurrent;
|
|
10789
|
+
// Fallback repair does not wait for the maybe-sync round trip.
|
|
10790
|
+
await this.pushRepairEntries(
|
|
10791
|
+
target,
|
|
10685
10792
|
unknownEntries,
|
|
10686
|
-
|
|
10687
|
-
|
|
10688
|
-
|
|
10689
|
-
|
|
10793
|
+
dispatchIsStillCurrent,
|
|
10794
|
+
options?.signal,
|
|
10795
|
+
);
|
|
10796
|
+
} else {
|
|
10797
|
+
const syncEntries = this._logProperties?.sync?.priority
|
|
10798
|
+
? (this._coordinates.materializeRepairDispatchEntries(
|
|
10799
|
+
unknownEntries,
|
|
10800
|
+
) as unknown as Map<string, SyncEntryCoordinates<R>>)
|
|
10801
|
+
: (unknownEntries as Map<string, SyncEntryCoordinates<R>>);
|
|
10802
|
+
if (!isStillCurrent()) return;
|
|
10803
|
+
await this.syncronizer.onMaybeMissingEntries({
|
|
10804
|
+
entries: syncEntries,
|
|
10805
|
+
targets: [target],
|
|
10806
|
+
signal: options?.signal,
|
|
10807
|
+
});
|
|
10808
|
+
}
|
|
10809
|
+
outcome = !lastObservedCurrent
|
|
10810
|
+
? "stale"
|
|
10811
|
+
: options?.signal?.aborted
|
|
10812
|
+
? "cancelled"
|
|
10813
|
+
: "dispatched";
|
|
10814
|
+
} catch (error) {
|
|
10815
|
+
outcome = "error";
|
|
10816
|
+
throw error;
|
|
10817
|
+
} finally {
|
|
10818
|
+
if (profile) {
|
|
10819
|
+
emitAdvisorySyncProfileDuration(profile, startedAt, {
|
|
10820
|
+
name: "sharedLog.repair.dispatch",
|
|
10821
|
+
component: "shared-log",
|
|
10822
|
+
entries: inputEntries,
|
|
10823
|
+
count: selectedEntries,
|
|
10824
|
+
targets: 1,
|
|
10825
|
+
details: {
|
|
10826
|
+
mode,
|
|
10827
|
+
transport,
|
|
10828
|
+
outcome,
|
|
10829
|
+
knownSuppressedEntries: knownHashes.length,
|
|
10830
|
+
},
|
|
10831
|
+
});
|
|
10832
|
+
}
|
|
10690
10833
|
}
|
|
10691
|
-
await this.syncronizer.onMaybeMissingEntries({
|
|
10692
|
-
entries: syncEntries,
|
|
10693
|
-
targets: [target],
|
|
10694
|
-
signal: options?.signal,
|
|
10695
|
-
});
|
|
10696
10834
|
}
|
|
10697
10835
|
|
|
10698
10836
|
private async sendMaybeMissingEntriesNow(
|
|
@@ -10776,6 +10914,7 @@ export class SharedLog<
|
|
|
10776
10914
|
this.isRepairLifecycleActive(repairLifecycleController),
|
|
10777
10915
|
signal: repairLifecycleController.signal,
|
|
10778
10916
|
},
|
|
10917
|
+
options.mode,
|
|
10779
10918
|
),
|
|
10780
10919
|
).catch((error: any) => logger.error(error));
|
|
10781
10920
|
}
|
|
@@ -11166,6 +11305,7 @@ export class SharedLog<
|
|
|
11166
11305
|
this.isRepairLifecycleActive(repairLifecycleController),
|
|
11167
11306
|
signal: repairLifecycleController.signal,
|
|
11168
11307
|
},
|
|
11308
|
+
options.mode,
|
|
11169
11309
|
),
|
|
11170
11310
|
).catch((error: any) => logger.error(error));
|
|
11171
11311
|
};
|
|
@@ -11309,6 +11449,18 @@ export class SharedLog<
|
|
|
11309
11449
|
repairLifecycleController: AbortController = this._instanceLifecycle
|
|
11310
11450
|
?.ownershipLifecycleController as AbortController,
|
|
11311
11451
|
) {
|
|
11452
|
+
const profile = this._logProperties?.sync?.profile;
|
|
11453
|
+
const startedAt = syncProfileStart(profile);
|
|
11454
|
+
const profileCounts = profile
|
|
11455
|
+
? {
|
|
11456
|
+
passes: 0,
|
|
11457
|
+
inputEntries: 0,
|
|
11458
|
+
nativePasses: 0,
|
|
11459
|
+
repairCandidates: 0,
|
|
11460
|
+
repairBatches: 0,
|
|
11461
|
+
outcome: "stale",
|
|
11462
|
+
}
|
|
11463
|
+
: undefined;
|
|
11312
11464
|
try {
|
|
11313
11465
|
while (this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
11314
11466
|
if (!this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
@@ -11347,8 +11499,10 @@ export class SharedLog<
|
|
|
11347
11499
|
pruneStaleJoinWarmupPeers();
|
|
11348
11500
|
|
|
11349
11501
|
if (pendingModes.size === 0) {
|
|
11502
|
+
if (profileCounts) profileCounts.outcome = "completed";
|
|
11350
11503
|
return;
|
|
11351
11504
|
}
|
|
11505
|
+
if (profileCounts) profileCounts.passes += 1;
|
|
11352
11506
|
|
|
11353
11507
|
const optimisticGidPeersByMode = new Map<
|
|
11354
11508
|
RepairDispatchMode,
|
|
@@ -11447,6 +11601,10 @@ export class SharedLog<
|
|
|
11447
11601
|
}
|
|
11448
11602
|
return;
|
|
11449
11603
|
}
|
|
11604
|
+
if (profileCounts) {
|
|
11605
|
+
profileCounts.repairCandidates += entries.size;
|
|
11606
|
+
profileCounts.repairBatches += 1;
|
|
11607
|
+
}
|
|
11450
11608
|
this.dispatchMaybeMissingEntries(
|
|
11451
11609
|
target,
|
|
11452
11610
|
entries,
|
|
@@ -11513,6 +11671,10 @@ export class SharedLog<
|
|
|
11513
11671
|
residentEntriesByHash &&
|
|
11514
11672
|
!this.hasCustomFindLeaders()
|
|
11515
11673
|
) {
|
|
11674
|
+
if (profileCounts) {
|
|
11675
|
+
profileCounts.nativePasses += 1;
|
|
11676
|
+
profileCounts.inputEntries += residentEntriesByHash.size;
|
|
11677
|
+
}
|
|
11516
11678
|
const repairDispatchPlan = pruneStaleJoinWarmupPeers()
|
|
11517
11679
|
? await this.planResidentRepairDispatchBatch(
|
|
11518
11680
|
{
|
|
@@ -11551,6 +11713,7 @@ export class SharedLog<
|
|
|
11551
11713
|
const entries = await iterator.next(
|
|
11552
11714
|
REPAIR_SWEEP_ENTRY_BATCH_SIZE,
|
|
11553
11715
|
);
|
|
11716
|
+
if (profileCounts) profileCounts.inputEntries += entries.length;
|
|
11554
11717
|
if (!this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
11555
11718
|
return;
|
|
11556
11719
|
}
|
|
@@ -11675,6 +11838,7 @@ export class SharedLog<
|
|
|
11675
11838
|
}
|
|
11676
11839
|
}
|
|
11677
11840
|
} catch (error: any) {
|
|
11841
|
+
if (profileCounts) profileCounts.outcome = "error";
|
|
11678
11842
|
if (
|
|
11679
11843
|
this.isRepairLifecycleActive(repairLifecycleController) &&
|
|
11680
11844
|
!isNotStartedError(error)
|
|
@@ -11696,6 +11860,21 @@ export class SharedLog<
|
|
|
11696
11860
|
void this.runRepairSweep(repairLifecycleController);
|
|
11697
11861
|
}
|
|
11698
11862
|
}
|
|
11863
|
+
if (profileCounts) {
|
|
11864
|
+
emitAdvisorySyncProfileDuration(profile, startedAt, {
|
|
11865
|
+
name: "sharedLog.placement.pass",
|
|
11866
|
+
component: "shared-log",
|
|
11867
|
+
entries: profileCounts.inputEntries,
|
|
11868
|
+
count: profileCounts.repairCandidates,
|
|
11869
|
+
details: {
|
|
11870
|
+
phase: "repair-sweep",
|
|
11871
|
+
outcome: profileCounts.outcome,
|
|
11872
|
+
passes: profileCounts.passes,
|
|
11873
|
+
nativePasses: profileCounts.nativePasses,
|
|
11874
|
+
repairBatches: profileCounts.repairBatches,
|
|
11875
|
+
},
|
|
11876
|
+
});
|
|
11877
|
+
}
|
|
11699
11878
|
}
|
|
11700
11879
|
}
|
|
11701
11880
|
|
|
@@ -20871,6 +21050,10 @@ export class SharedLog<
|
|
|
20871
21050
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
20872
21051
|
try {
|
|
20873
21052
|
this.stopSubscriptionChangeCallbackAdmission();
|
|
21053
|
+
// A receive may be awaiting a synchronizer response shipment. Cancel
|
|
21054
|
+
// its dispatch generation before draining that receive, rather than
|
|
21055
|
+
// waiting for _close() to reach the synchronizer's final teardown.
|
|
21056
|
+
this.syncronizer?.beginClose?.();
|
|
20874
21057
|
this.joinWarmup.cancelAllJoinWarmupTargets();
|
|
20875
21058
|
await this.drainSubscriptionChangeCallbacks();
|
|
20876
21059
|
// An already-admitted subscription callback can create a fresh warmup
|
|
@@ -21009,6 +21192,7 @@ export class SharedLog<
|
|
|
21009
21192
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
21010
21193
|
try {
|
|
21011
21194
|
this.stopSubscriptionChangeCallbackAdmission();
|
|
21195
|
+
this.syncronizer?.beginClose?.();
|
|
21012
21196
|
this.joinWarmup.cancelAllJoinWarmupTargets();
|
|
21013
21197
|
await this.drainSubscriptionChangeCallbacks();
|
|
21014
21198
|
// An already-admitted subscription callback can create a fresh warmup
|
|
@@ -21274,10 +21458,11 @@ export class SharedLog<
|
|
|
21274
21458
|
msg.heads.map((head) => head.hash),
|
|
21275
21459
|
);
|
|
21276
21460
|
if (syncProfile) {
|
|
21277
|
-
|
|
21461
|
+
emitAdvisorySyncProfileDuration(syncProfile, rawExistingStartedAt, {
|
|
21278
21462
|
name: "sharedLog.rawReceive.existingHeads",
|
|
21279
21463
|
component: "shared-log",
|
|
21280
21464
|
entries: msg.heads.length,
|
|
21465
|
+
count: rawExistingHashes.size,
|
|
21281
21466
|
messages: 1,
|
|
21282
21467
|
});
|
|
21283
21468
|
}
|
|
@@ -21711,10 +21896,11 @@ export class SharedLog<
|
|
|
21711
21896
|
? undefined
|
|
21712
21897
|
: await this.log.hasMany(headHashes);
|
|
21713
21898
|
if (syncProfile) {
|
|
21714
|
-
|
|
21899
|
+
emitAdvisorySyncProfileDuration(syncProfile, existingStartedAt, {
|
|
21715
21900
|
name: "sharedLog.receive.existingHeads",
|
|
21716
21901
|
component: "shared-log",
|
|
21717
21902
|
entries: heads.length,
|
|
21903
|
+
count: existingHashes?.size,
|
|
21718
21904
|
messages: 1,
|
|
21719
21905
|
details: { rawMaterializedKnownMissing },
|
|
21720
21906
|
});
|
|
@@ -29478,6 +29664,17 @@ export class SharedLog<
|
|
|
29478
29664
|
isOwnershipLifecycleCurrent() &&
|
|
29479
29665
|
[...warmupPeers].every(isCurrentJoinWarmupTarget);
|
|
29480
29666
|
|
|
29667
|
+
const profile = this._logProperties?.sync?.profile;
|
|
29668
|
+
const profileStartedAt = syncProfileStart(profile);
|
|
29669
|
+
const profileCounts = profile
|
|
29670
|
+
? {
|
|
29671
|
+
examinedEntries: 0,
|
|
29672
|
+
repairCandidates: 0,
|
|
29673
|
+
repairBatches: 0,
|
|
29674
|
+
pruneScan: false,
|
|
29675
|
+
outcome: "stale",
|
|
29676
|
+
}
|
|
29677
|
+
: undefined;
|
|
29481
29678
|
try {
|
|
29482
29679
|
const uncheckedDeliver: Map<
|
|
29483
29680
|
string,
|
|
@@ -29501,6 +29698,10 @@ export class SharedLog<
|
|
|
29501
29698
|
: isWarmupTarget
|
|
29502
29699
|
? "join-warmup"
|
|
29503
29700
|
: "join-authoritative";
|
|
29701
|
+
if (profileCounts) {
|
|
29702
|
+
profileCounts.repairCandidates += entries.size;
|
|
29703
|
+
profileCounts.repairBatches += 1;
|
|
29704
|
+
}
|
|
29504
29705
|
this.dispatchMaybeMissingEntries(
|
|
29505
29706
|
target,
|
|
29506
29707
|
entries,
|
|
@@ -29555,6 +29756,7 @@ export class SharedLog<
|
|
|
29555
29756
|
forceFresh: forceFreshDelivery || useJoinWarmupFastPath,
|
|
29556
29757
|
},
|
|
29557
29758
|
)) {
|
|
29759
|
+
if (profileCounts) profileCounts.examinedEntries += 1;
|
|
29558
29760
|
if (
|
|
29559
29761
|
!isOwnershipLifecycleCurrent() ||
|
|
29560
29762
|
(useJoinWarmupFastPath && !areJoinWarmupGenerationsCurrent())
|
|
@@ -29838,6 +30040,7 @@ export class SharedLog<
|
|
|
29838
30040
|
));
|
|
29839
30041
|
|
|
29840
30042
|
if (shouldRunLocalPruneScan) {
|
|
30043
|
+
if (profileCounts) profileCounts.pruneScan = true;
|
|
29841
30044
|
throwIfOwnershipLifecycleInactive();
|
|
29842
30045
|
// Adaptive range changes and fixed zero-width updates can make already-indexed
|
|
29843
30046
|
// local heads prunable even when the incremental rebalance scan misses them
|
|
@@ -29857,6 +30060,7 @@ export class SharedLog<
|
|
|
29857
30060
|
}
|
|
29858
30061
|
}
|
|
29859
30062
|
|
|
30063
|
+
if (profileCounts) profileCounts.outcome = "completed";
|
|
29860
30064
|
return changed;
|
|
29861
30065
|
} catch (error: any) {
|
|
29862
30066
|
if (!isOwnershipLifecycleCurrent()) {
|
|
@@ -29866,8 +30070,27 @@ export class SharedLog<
|
|
|
29866
30070
|
return false; // we are not started yet, so no changes
|
|
29867
30071
|
}
|
|
29868
30072
|
|
|
30073
|
+
if (profileCounts) profileCounts.outcome = "error";
|
|
29869
30074
|
logger.error(error.toString());
|
|
29870
30075
|
throw error;
|
|
30076
|
+
} finally {
|
|
30077
|
+
if (profileCounts) {
|
|
30078
|
+
emitAdvisorySyncProfileDuration(profile, profileStartedAt, {
|
|
30079
|
+
name: "sharedLog.placement.pass",
|
|
30080
|
+
component: "shared-log",
|
|
30081
|
+
entries: profileCounts.examinedEntries,
|
|
30082
|
+
count: profileCounts.repairCandidates,
|
|
30083
|
+
details: {
|
|
30084
|
+
phase: "range-change",
|
|
30085
|
+
outcome: profileCounts.outcome,
|
|
30086
|
+
changes: changes.length,
|
|
30087
|
+
repairBatches: profileCounts.repairBatches,
|
|
30088
|
+
pruneScan: profileCounts.pruneScan,
|
|
30089
|
+
forceFreshDelivery,
|
|
30090
|
+
joinWarmupFastPath: useJoinWarmupFastPath,
|
|
30091
|
+
},
|
|
30092
|
+
});
|
|
30093
|
+
}
|
|
29871
30094
|
}
|
|
29872
30095
|
}
|
|
29873
30096
|
|
|
@@ -29926,6 +30149,15 @@ export class SharedLog<
|
|
|
29926
30149
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
29927
30150
|
rebalanceParticipationDebounced = this.rebalanceParticipationDebounced,
|
|
29928
30151
|
) {
|
|
30152
|
+
const profile = this._isAdaptiveReplicating
|
|
30153
|
+
? this._logProperties?.sync?.profile
|
|
30154
|
+
: undefined;
|
|
30155
|
+
const profileStartedAt = syncProfileStart(profile);
|
|
30156
|
+
const profileDetails:
|
|
30157
|
+
| Record<string, string | number | boolean | undefined>
|
|
30158
|
+
| undefined = profile
|
|
30159
|
+
? { outcome: "stale", idleRemainingMs: 0 }
|
|
30160
|
+
: undefined;
|
|
29929
30161
|
// Stage 3: the lifecycle owns all three identity terms. `lifecycle` may
|
|
29930
30162
|
// go stale later; its deps late-bind to the host, so the debouncer term
|
|
29931
30163
|
// still reads the current host field, and the role term can disagree
|
|
@@ -29965,6 +30197,14 @@ export class SharedLog<
|
|
|
29965
30197
|
|
|
29966
30198
|
if (this._isAdaptiveReplicating) {
|
|
29967
30199
|
if (this.shouldDelayAdaptiveRebalance()) {
|
|
30200
|
+
if (profileDetails) {
|
|
30201
|
+
profileDetails.outcome = "idle-deferred";
|
|
30202
|
+
profileDetails.idleRemainingMs = Math.max(
|
|
30203
|
+
0,
|
|
30204
|
+
this.adaptiveRebalanceIdleMs -
|
|
30205
|
+
(Date.now() - this._lastLocalAppendAt),
|
|
30206
|
+
);
|
|
30207
|
+
}
|
|
29968
30208
|
if (isCurrent()) {
|
|
29969
30209
|
void rebalanceParticipationDebounced?.call();
|
|
29970
30210
|
}
|
|
@@ -29974,11 +30214,17 @@ export class SharedLog<
|
|
|
29974
30214
|
const peers = this.replicationIndex;
|
|
29975
30215
|
const usedMemory = await this.getMemoryUsage();
|
|
29976
30216
|
if (!isCurrent()) return false;
|
|
30217
|
+
if (profileDetails) {
|
|
30218
|
+
profileDetails.storageUsedBytes = usedMemory;
|
|
30219
|
+
profileDetails.storageObjectiveBytes =
|
|
30220
|
+
this.replicationController.maxMemoryLimit;
|
|
30221
|
+
}
|
|
29977
30222
|
this.scheduleReplicationStatusRefreshForStorage(usedMemory);
|
|
29978
30223
|
let dynamicRange = await this.getDynamicRange();
|
|
29979
30224
|
if (!isCurrent()) return false;
|
|
29980
30225
|
|
|
29981
30226
|
if (!dynamicRange) {
|
|
30227
|
+
if (profileDetails) profileDetails.outcome = "not-permitted";
|
|
29982
30228
|
return; // not allowed to replicate
|
|
29983
30229
|
}
|
|
29984
30230
|
|
|
@@ -30005,13 +30251,24 @@ export class SharedLog<
|
|
|
30005
30251
|
const totalParticipation = await this.calculateTotalParticipation();
|
|
30006
30252
|
if (!isCurrent()) return false;
|
|
30007
30253
|
|
|
30254
|
+
const cpuUsage = this.cpuUsage?.value();
|
|
30255
|
+
const stepStartedAt = syncProfileStart(profile);
|
|
30008
30256
|
const newFactor = this.replicationController.step({
|
|
30009
30257
|
memoryUsage: usedMemory,
|
|
30010
30258
|
currentFactor: dynamicRange.widthNormalized,
|
|
30011
30259
|
totalFactor: totalParticipation, // TODO use this._totalParticipation when flakiness is fixed
|
|
30012
30260
|
peerCount: peersSize,
|
|
30013
|
-
cpuUsage
|
|
30261
|
+
cpuUsage,
|
|
30014
30262
|
});
|
|
30263
|
+
if (profileDetails) {
|
|
30264
|
+
profileDetails.preStepMs = stepStartedAt - profileStartedAt;
|
|
30265
|
+
profileDetails.stepMs = syncProfileStart(profile) - stepStartedAt;
|
|
30266
|
+
profileDetails.currentFactor = dynamicRange.widthNormalized;
|
|
30267
|
+
profileDetails.proposedFactor = newFactor;
|
|
30268
|
+
profileDetails.totalFactor = totalParticipation;
|
|
30269
|
+
profileDetails.controllerPeerCount = peersSize;
|
|
30270
|
+
profileDetails.cpuUsage = cpuUsage;
|
|
30271
|
+
}
|
|
30015
30272
|
|
|
30016
30273
|
const absoluteDifference = Math.abs(
|
|
30017
30274
|
dynamicRange.widthNormalized - newFactor,
|
|
@@ -30048,9 +30305,11 @@ export class SharedLog<
|
|
|
30048
30305
|
(await this._isTrustedReplicator(this.node.identity.publicKey));
|
|
30049
30306
|
if (!isCurrent()) return false;
|
|
30050
30307
|
if (!canReplicate) {
|
|
30308
|
+
if (profileDetails) profileDetails.outcome = "not-permitted";
|
|
30051
30309
|
return false;
|
|
30052
30310
|
}
|
|
30053
30311
|
|
|
30312
|
+
const applyStartedAt = syncProfileStart(profile);
|
|
30054
30313
|
await this.startAnnounceReplicating(
|
|
30055
30314
|
[dynamicRange],
|
|
30056
30315
|
{
|
|
@@ -30061,6 +30320,10 @@ export class SharedLog<
|
|
|
30061
30320
|
ownershipLifecycleController,
|
|
30062
30321
|
);
|
|
30063
30322
|
if (!isCurrent()) return false;
|
|
30323
|
+
if (profileDetails) {
|
|
30324
|
+
profileDetails.outcome = "apply-settled";
|
|
30325
|
+
profileDetails.applyMs = syncProfileStart(profile) - applyStartedAt;
|
|
30326
|
+
}
|
|
30064
30327
|
|
|
30065
30328
|
/* await this._updateRole(newRole, onRoleChange); */
|
|
30066
30329
|
if (isCurrent()) {
|
|
@@ -30069,6 +30332,7 @@ export class SharedLog<
|
|
|
30069
30332
|
|
|
30070
30333
|
return true;
|
|
30071
30334
|
} else {
|
|
30335
|
+
if (profileDetails) profileDetails.outcome = "unchanged";
|
|
30072
30336
|
if (isCurrent()) {
|
|
30073
30337
|
void rebalanceParticipationDebounced?.call();
|
|
30074
30338
|
}
|
|
@@ -30078,14 +30342,24 @@ export class SharedLog<
|
|
|
30078
30342
|
return false;
|
|
30079
30343
|
};
|
|
30080
30344
|
|
|
30081
|
-
|
|
30082
|
-
|
|
30083
|
-
|
|
30345
|
+
try {
|
|
30346
|
+
return await fn().catch((error: any) => {
|
|
30347
|
+
if (isNotStartedError(error) || isClosedStoreRace(error)) {
|
|
30348
|
+
if (profileDetails) profileDetails.outcome = "stale";
|
|
30349
|
+
return false;
|
|
30350
|
+
}
|
|
30351
|
+
if (profileDetails) profileDetails.outcome = "error";
|
|
30352
|
+
throw error;
|
|
30353
|
+
});
|
|
30354
|
+
} finally {
|
|
30355
|
+
if (profileDetails) {
|
|
30356
|
+
emitAdvisorySyncProfileDuration(profile, profileStartedAt, {
|
|
30357
|
+
name: "sharedLog.adaptive.rebalance",
|
|
30358
|
+
component: "shared-log",
|
|
30359
|
+
details: profileDetails,
|
|
30360
|
+
});
|
|
30084
30361
|
}
|
|
30085
|
-
|
|
30086
|
-
});
|
|
30087
|
-
|
|
30088
|
-
return resp;
|
|
30362
|
+
}
|
|
30089
30363
|
}
|
|
30090
30364
|
|
|
30091
30365
|
private getDynamicRangeOffset(): NumberFromType<R> {
|
package/src/sync/index.ts
CHANGED
|
@@ -251,6 +251,13 @@ export interface Syncronizer<R extends "u32" | "u64"> {
|
|
|
251
251
|
onPeerDisconnected(key: PublicSignKey | string): void;
|
|
252
252
|
|
|
253
253
|
open(): Promise<void> | void;
|
|
254
|
+
/**
|
|
255
|
+
* Synchronously fence dispatch admission and cancel the current generation's
|
|
256
|
+
* network waits before SharedLog drains admitted receives. Do not stop shared
|
|
257
|
+
* indexes or discard physical-work accounting here: close() still runs after
|
|
258
|
+
* those receives settle. Optional for existing custom synchronizers.
|
|
259
|
+
*/
|
|
260
|
+
beginClose?(): void;
|
|
254
261
|
close(): Promise<void> | void;
|
|
255
262
|
|
|
256
263
|
get pending(): number;
|
|
@@ -1127,6 +1127,9 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
1127
1127
|
timeoutMs?: number;
|
|
1128
1128
|
retryIntervalsMs?: number[];
|
|
1129
1129
|
}): RepairSession {
|
|
1130
|
+
if (this.ratelessClosed) {
|
|
1131
|
+
return this.simple.startRepairSession(properties);
|
|
1132
|
+
}
|
|
1130
1133
|
const mode = properties.mode ?? "best-effort";
|
|
1131
1134
|
const targets = [...new Set(properties.targets)];
|
|
1132
1135
|
const timeoutMs = Math.max(
|
|
@@ -3303,13 +3306,16 @@ export class RatelessIBLTSynchronizer<D extends "u32" | "u64">
|
|
|
3303
3306
|
return this.simple.open();
|
|
3304
3307
|
}
|
|
3305
3308
|
|
|
3306
|
-
|
|
3309
|
+
beginClose(): void {
|
|
3307
3310
|
this.ratelessClosed = true;
|
|
3308
|
-
// Abort ownership first. Process abort listeners then cancel any in-flight
|
|
3309
|
-
// StartSync/MoreSymbols send before they detach or free native state.
|
|
3310
3311
|
const reason = new Error("rateless synchronizer closed");
|
|
3311
3312
|
this.cancelRatelessRepairSessions(reason);
|
|
3312
3313
|
this.ratelessDispatchLifecycleController.abort(reason);
|
|
3314
|
+
this.simple.beginClose();
|
|
3315
|
+
}
|
|
3316
|
+
|
|
3317
|
+
close(): Promise<void> | void {
|
|
3318
|
+
this.beginClose();
|
|
3313
3319
|
for (const obj of [...this.ingoingSyncProcesses.values()]) {
|
|
3314
3320
|
obj.free();
|
|
3315
3321
|
}
|