@peerbit/shared-log 16.0.29 → 16.0.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +306 -42
- package/dist/src/index.js.map +1 -1
- package/dist/src/sync/index.d.ts +7 -0
- package/dist/src/sync/index.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.d.ts +1 -0
- package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.js +8 -3
- package/dist/src/sync/rateless-iblt.js.map +1 -1
- package/dist/src/sync/simple.d.ts +1 -0
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +16 -3
- package/dist/src/sync/simple.js.map +1 -1
- package/package.json +15 -15
- package/src/index.ts +330 -56
- package/src/sync/index.ts +7 -0
- package/src/sync/rateless-iblt.ts +9 -3
- package/src/sync/simple.ts +17 -3
package/dist/src/index.js
CHANGED
|
@@ -186,7 +186,7 @@ const emitAdvisorySyncProfileDuration = (profile, startedAt, event) => {
|
|
|
186
186
|
emitSyncProfileDuration(profile, startedAt, event);
|
|
187
187
|
}
|
|
188
188
|
catch {
|
|
189
|
-
//
|
|
189
|
+
// Advisory diagnostics must not change replication or lifecycle behavior.
|
|
190
190
|
}
|
|
191
191
|
};
|
|
192
192
|
const canUseOptionalNativeModuleImports = () => {
|
|
@@ -1964,7 +1964,7 @@ let SharedLog = (() => {
|
|
|
1964
1964
|
bumpSimpleFallbackPasses: () => {
|
|
1965
1965
|
this._repairMetrics["join-warmup"].simpleFallbackPasses += 1;
|
|
1966
1966
|
},
|
|
1967
|
-
sendEntriesSimple: (target, entries, options) => this.sendRepairEntriesWithTransport(target, entries, "simple", options),
|
|
1967
|
+
sendEntriesSimple: (target, entries, options) => this.sendRepairEntriesWithTransport(target, entries, "simple", options, "join-warmup"),
|
|
1968
1968
|
logError: (error) => logger.error(error),
|
|
1969
1969
|
});
|
|
1970
1970
|
}
|
|
@@ -3694,6 +3694,76 @@ let SharedLog = (() => {
|
|
|
3694
3694
|
const deadline = persistedDeadline ??
|
|
3695
3695
|
this.createPersistedDeliveryDeadline(delivery, ownershipLifecycleController, records.size);
|
|
3696
3696
|
const signal = deadline.signal;
|
|
3697
|
+
const recoveryController = new AbortController();
|
|
3698
|
+
const recoverySignal = AbortSignal.any([signal, recoveryController.signal]);
|
|
3699
|
+
const recoveryByPeer = new Map();
|
|
3700
|
+
let recoveryCursor = 0;
|
|
3701
|
+
const recoverSelectedPeers = (selected) => {
|
|
3702
|
+
// Recovery is advisory work, not a receipt or a replacement leader plan.
|
|
3703
|
+
// Never occupy transfer/request slots while waiting for it. The separate
|
|
3704
|
+
// bounded pool rotates through fresh candidates so quiet/incomplete peers
|
|
3705
|
+
// cannot indefinitely hide a later recoverable peer.
|
|
3706
|
+
for (const [peer, state] of recoveryByPeer) {
|
|
3707
|
+
if (!selected.has(peer)) {
|
|
3708
|
+
clearTimeout(state.timer);
|
|
3709
|
+
state.controller.abort();
|
|
3710
|
+
}
|
|
3711
|
+
}
|
|
3712
|
+
const peers = [...selected];
|
|
3713
|
+
for (let visited = 0; visited < peers.length &&
|
|
3714
|
+
recoveryByPeer.size < MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL &&
|
|
3715
|
+
!recoverySignal.aborted; visited++) {
|
|
3716
|
+
const peer = peers[recoveryCursor++ % peers.length];
|
|
3717
|
+
if (recoveryByPeer.has(peer))
|
|
3718
|
+
continue;
|
|
3719
|
+
const current = this.persistedReceiptPeerSession(peer);
|
|
3720
|
+
if (current &&
|
|
3721
|
+
this._v2Send.isLatestConfirmedForPeer({
|
|
3722
|
+
peerHash: peer,
|
|
3723
|
+
peerSession: current.peerSession,
|
|
3724
|
+
receiverTransportSession: current.capabilitySession,
|
|
3725
|
+
})) {
|
|
3726
|
+
continue;
|
|
3727
|
+
}
|
|
3728
|
+
const expiresAt = Math.min(deadline.deadline, Date.now() + MAX_PERSISTED_RECEIPT_ATTEMPT_MS);
|
|
3729
|
+
const controller = new AbortController();
|
|
3730
|
+
const attemptSignal = AbortSignal.any([
|
|
3731
|
+
recoverySignal,
|
|
3732
|
+
controller.signal,
|
|
3733
|
+
]);
|
|
3734
|
+
const timer = setTimeout(() => controller.abort(), Math.max(1, expiresAt - Date.now()));
|
|
3735
|
+
timer.unref?.();
|
|
3736
|
+
const state = { controller, timer };
|
|
3737
|
+
recoveryByPeer.set(peer, state);
|
|
3738
|
+
void (async () => {
|
|
3739
|
+
// Resolve only an authenticated transport-cache key. Keep this slot
|
|
3740
|
+
// reserved until the lookup actually settles, even if a custom
|
|
3741
|
+
// resolver ignores cancellation, rather than launching duplicates.
|
|
3742
|
+
const key = await this._resolvePublicKeyFromHash(peer);
|
|
3743
|
+
if (attemptSignal.aborted ||
|
|
3744
|
+
!key ||
|
|
3745
|
+
key.hashcode() !== peer ||
|
|
3746
|
+
Date.now() >= expiresAt) {
|
|
3747
|
+
return;
|
|
3748
|
+
}
|
|
3749
|
+
// Reuse the exact-session watchdog, capability/subscriber recovery,
|
|
3750
|
+
// and replacement-session handling from the public preflight. Its
|
|
3751
|
+
// result is ignored: settlement still checks the exact entry leaders
|
|
3752
|
+
// and accepts only the subsequent durable, session-bound receipts.
|
|
3753
|
+
await this.waitForPersistedReceiptPeerReadiness(key, {
|
|
3754
|
+
timeout: Math.max(1, expiresAt - Date.now()),
|
|
3755
|
+
signal: attemptSignal,
|
|
3756
|
+
});
|
|
3757
|
+
})()
|
|
3758
|
+
.catch(() => undefined)
|
|
3759
|
+
.finally(() => {
|
|
3760
|
+
clearTimeout(timer);
|
|
3761
|
+
if (recoveryByPeer.get(peer) === state) {
|
|
3762
|
+
recoveryByPeer.delete(peer);
|
|
3763
|
+
}
|
|
3764
|
+
});
|
|
3765
|
+
}
|
|
3766
|
+
};
|
|
3697
3767
|
let maxAttemptMs = MAX_PERSISTED_RECEIPT_ATTEMPT_MS;
|
|
3698
3768
|
let initialTransferPending = transferOnFirstRound;
|
|
3699
3769
|
let needsInitialLeaderCheck = true;
|
|
@@ -3741,6 +3811,7 @@ let SharedLog = (() => {
|
|
|
3741
3811
|
// transport epoch. A revision/session change purges them before they can
|
|
3742
3812
|
// survive an away-and-back leader transition or combine with a later peer.
|
|
3743
3813
|
const hashesByPeer = new Map();
|
|
3814
|
+
const recoveryCandidates = new Set();
|
|
3744
3815
|
const entryArray = [...records.values()];
|
|
3745
3816
|
const leadersByEntry = await this.planPersistedDeliveryLeaders(entryArray, replicas, ownershipLifecycleController);
|
|
3746
3817
|
if (!isRoundOwnershipCurrent())
|
|
@@ -3779,6 +3850,7 @@ let SharedLog = (() => {
|
|
|
3779
3850
|
for (const peer of leaders.keys()) {
|
|
3780
3851
|
if (peer === selfHash)
|
|
3781
3852
|
continue;
|
|
3853
|
+
recoveryCandidates.add(peer);
|
|
3782
3854
|
const current = this.persistedReceiptPeerSession(peer);
|
|
3783
3855
|
if (!current)
|
|
3784
3856
|
continue;
|
|
@@ -3791,6 +3863,7 @@ let SharedLog = (() => {
|
|
|
3791
3863
|
}
|
|
3792
3864
|
if (!isRoundOwnershipCurrent())
|
|
3793
3865
|
continue;
|
|
3866
|
+
recoverSelectedPeers(recoveryCandidates);
|
|
3794
3867
|
const operationQueue = new PQueue({
|
|
3795
3868
|
concurrency: MAX_PERSISTED_RECEIPT_REQUESTS_GLOBAL,
|
|
3796
3869
|
});
|
|
@@ -4100,6 +4173,11 @@ let SharedLog = (() => {
|
|
|
4100
4173
|
throw new PersistedDeliveryError(error, committedHashes);
|
|
4101
4174
|
}
|
|
4102
4175
|
finally {
|
|
4176
|
+
recoveryController.abort();
|
|
4177
|
+
for (const state of recoveryByPeer.values()) {
|
|
4178
|
+
clearTimeout(state.timer);
|
|
4179
|
+
state.controller.abort();
|
|
4180
|
+
}
|
|
4103
4181
|
if (ownedDeadline)
|
|
4104
4182
|
deadline.dispose();
|
|
4105
4183
|
}
|
|
@@ -6857,47 +6935,88 @@ let SharedLog = (() => {
|
|
|
6857
6935
|
signal,
|
|
6858
6936
|
});
|
|
6859
6937
|
}
|
|
6860
|
-
async sendRepairEntriesWithTransport(target, entries, transport, options) {
|
|
6938
|
+
async sendRepairEntriesWithTransport(target, entries, transport, options, mode) {
|
|
6861
6939
|
const isStillCurrent = options?.isStillCurrent ?? (() => true);
|
|
6862
6940
|
if (!isStillCurrent()) {
|
|
6863
6941
|
return;
|
|
6864
6942
|
}
|
|
6865
6943
|
const unknownEntries = new Map();
|
|
6866
6944
|
const knownHashes = [];
|
|
6867
|
-
|
|
6868
|
-
|
|
6869
|
-
|
|
6870
|
-
|
|
6871
|
-
|
|
6945
|
+
const profile = this._logProperties?.sync?.profile;
|
|
6946
|
+
const startedAt = syncProfileStart(profile);
|
|
6947
|
+
const inputEntries = profile ? entries.size : 0;
|
|
6948
|
+
let selectedEntries = 0;
|
|
6949
|
+
let lastObservedCurrent = true;
|
|
6950
|
+
let outcome = "stale";
|
|
6951
|
+
try {
|
|
6952
|
+
for (const [hash, entry] of entries) {
|
|
6953
|
+
if ((options?.bypassRecentKnownPeers ||
|
|
6954
|
+
!this.isEntryRecentlyKnownByPeer(hash, target, RECENT_KNOWN_REPAIR_SUPPRESSION_MS)) &&
|
|
6955
|
+
(options?.bypassKnownPeers || !this.isEntryKnownByPeer(hash, target))) {
|
|
6956
|
+
unknownEntries.set(hash, entry);
|
|
6957
|
+
}
|
|
6958
|
+
else {
|
|
6959
|
+
knownHashes.push(hash);
|
|
6960
|
+
}
|
|
6961
|
+
}
|
|
6962
|
+
// A custom synchronizer may mutate the Map once it receives it.
|
|
6963
|
+
if (profile)
|
|
6964
|
+
selectedEntries = unknownEntries.size;
|
|
6965
|
+
if (!isStillCurrent())
|
|
6966
|
+
return;
|
|
6967
|
+
this.clearRepairFrontierHashes(target, knownHashes);
|
|
6968
|
+
if (unknownEntries.size === 0) {
|
|
6969
|
+
outcome = "known-suppressed";
|
|
6970
|
+
return;
|
|
6971
|
+
}
|
|
6972
|
+
if (transport === "simple") {
|
|
6973
|
+
// Observe only checks the lower path already makes, without adding
|
|
6974
|
+
// lifecycle decisions or wrapping the disabled-profiling path.
|
|
6975
|
+
const dispatchIsStillCurrent = profile
|
|
6976
|
+
? () => (lastObservedCurrent = isStillCurrent())
|
|
6977
|
+
: isStillCurrent;
|
|
6978
|
+
// Fallback repair does not wait for the maybe-sync round trip.
|
|
6979
|
+
await this.pushRepairEntries(target, unknownEntries, dispatchIsStillCurrent, options?.signal);
|
|
6872
6980
|
}
|
|
6873
6981
|
else {
|
|
6874
|
-
|
|
6982
|
+
const syncEntries = this._logProperties?.sync?.priority
|
|
6983
|
+
? this._coordinates.materializeRepairDispatchEntries(unknownEntries)
|
|
6984
|
+
: unknownEntries;
|
|
6985
|
+
if (!isStillCurrent())
|
|
6986
|
+
return;
|
|
6987
|
+
await this.syncronizer.onMaybeMissingEntries({
|
|
6988
|
+
entries: syncEntries,
|
|
6989
|
+
targets: [target],
|
|
6990
|
+
signal: options?.signal,
|
|
6991
|
+
});
|
|
6875
6992
|
}
|
|
6993
|
+
outcome = !lastObservedCurrent
|
|
6994
|
+
? "stale"
|
|
6995
|
+
: options?.signal?.aborted
|
|
6996
|
+
? "cancelled"
|
|
6997
|
+
: "dispatched";
|
|
6876
6998
|
}
|
|
6877
|
-
|
|
6878
|
-
|
|
6879
|
-
|
|
6880
|
-
this.clearRepairFrontierHashes(target, knownHashes);
|
|
6881
|
-
if (unknownEntries.size === 0) {
|
|
6882
|
-
return;
|
|
6883
|
-
}
|
|
6884
|
-
if (transport === "simple") {
|
|
6885
|
-
// Fallback repair should not depend on the target completing the
|
|
6886
|
-
// RequestMaybeSync -> ResponseMaybeSync round trip.
|
|
6887
|
-
await this.pushRepairEntries(target, unknownEntries, isStillCurrent, options?.signal);
|
|
6888
|
-
return;
|
|
6999
|
+
catch (error) {
|
|
7000
|
+
outcome = "error";
|
|
7001
|
+
throw error;
|
|
6889
7002
|
}
|
|
6890
|
-
|
|
6891
|
-
|
|
6892
|
-
|
|
6893
|
-
|
|
6894
|
-
|
|
7003
|
+
finally {
|
|
7004
|
+
if (profile) {
|
|
7005
|
+
emitAdvisorySyncProfileDuration(profile, startedAt, {
|
|
7006
|
+
name: "sharedLog.repair.dispatch",
|
|
7007
|
+
component: "shared-log",
|
|
7008
|
+
entries: inputEntries,
|
|
7009
|
+
count: selectedEntries,
|
|
7010
|
+
targets: 1,
|
|
7011
|
+
details: {
|
|
7012
|
+
mode,
|
|
7013
|
+
transport,
|
|
7014
|
+
outcome,
|
|
7015
|
+
knownSuppressedEntries: knownHashes.length,
|
|
7016
|
+
},
|
|
7017
|
+
});
|
|
7018
|
+
}
|
|
6895
7019
|
}
|
|
6896
|
-
await this.syncronizer.onMaybeMissingEntries({
|
|
6897
|
-
entries: syncEntries,
|
|
6898
|
-
targets: [target],
|
|
6899
|
-
signal: options?.signal,
|
|
6900
|
-
});
|
|
6901
7020
|
}
|
|
6902
7021
|
async sendMaybeMissingEntriesNow(target, entries, options, repairLifecycleController = this._instanceLifecycle
|
|
6903
7022
|
?.ownershipLifecycleController) {
|
|
@@ -6955,7 +7074,7 @@ let SharedLog = (() => {
|
|
|
6955
7074
|
bypassRecentKnownPeers: bypassKnownPeerHints,
|
|
6956
7075
|
isStillCurrent: () => this.isRepairLifecycleActive(repairLifecycleController),
|
|
6957
7076
|
signal: repairLifecycleController.signal,
|
|
6958
|
-
})).catch((error) => logger.error(error));
|
|
7077
|
+
}, options.mode)).catch((error) => logger.error(error));
|
|
6959
7078
|
}
|
|
6960
7079
|
ensureRepairFrontierRunner(mode, target, retryScheduleMs, repairLifecycleController = this._instanceLifecycle
|
|
6961
7080
|
?.ownershipLifecycleController) {
|
|
@@ -7215,7 +7334,7 @@ let SharedLog = (() => {
|
|
|
7215
7334
|
bypassRecentKnownPeers: bypassKnownPeerHints,
|
|
7216
7335
|
isStillCurrent: () => this.isRepairLifecycleActive(repairLifecycleController),
|
|
7217
7336
|
signal: repairLifecycleController.signal,
|
|
7218
|
-
})).catch((error) => logger.error(error));
|
|
7337
|
+
}, options.mode)).catch((error) => logger.error(error));
|
|
7219
7338
|
};
|
|
7220
7339
|
const delayedJoinWarmupRetries = [];
|
|
7221
7340
|
retrySchedule.forEach((delayMs, index) => {
|
|
@@ -7320,6 +7439,18 @@ let SharedLog = (() => {
|
|
|
7320
7439
|
}
|
|
7321
7440
|
async runRepairSweep(repairLifecycleController = this._instanceLifecycle
|
|
7322
7441
|
?.ownershipLifecycleController) {
|
|
7442
|
+
const profile = this._logProperties?.sync?.profile;
|
|
7443
|
+
const startedAt = syncProfileStart(profile);
|
|
7444
|
+
const profileCounts = profile
|
|
7445
|
+
? {
|
|
7446
|
+
passes: 0,
|
|
7447
|
+
inputEntries: 0,
|
|
7448
|
+
nativePasses: 0,
|
|
7449
|
+
repairCandidates: 0,
|
|
7450
|
+
repairBatches: 0,
|
|
7451
|
+
outcome: "stale",
|
|
7452
|
+
}
|
|
7453
|
+
: undefined;
|
|
7323
7454
|
try {
|
|
7324
7455
|
while (this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
7325
7456
|
if (!this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
@@ -7351,8 +7482,12 @@ let SharedLog = (() => {
|
|
|
7351
7482
|
};
|
|
7352
7483
|
pruneStaleJoinWarmupPeers();
|
|
7353
7484
|
if (pendingModes.size === 0) {
|
|
7485
|
+
if (profileCounts)
|
|
7486
|
+
profileCounts.outcome = "completed";
|
|
7354
7487
|
return;
|
|
7355
7488
|
}
|
|
7489
|
+
if (profileCounts)
|
|
7490
|
+
profileCounts.passes += 1;
|
|
7356
7491
|
const optimisticGidPeersByMode = new Map();
|
|
7357
7492
|
const optimisticGidPeersConsumedByMode = new Map();
|
|
7358
7493
|
for (const mode of pendingModes) {
|
|
@@ -7423,6 +7558,10 @@ let SharedLog = (() => {
|
|
|
7423
7558
|
}
|
|
7424
7559
|
return;
|
|
7425
7560
|
}
|
|
7561
|
+
if (profileCounts) {
|
|
7562
|
+
profileCounts.repairCandidates += entries.size;
|
|
7563
|
+
profileCounts.repairBatches += 1;
|
|
7564
|
+
}
|
|
7426
7565
|
this.dispatchMaybeMissingEntries(target, entries, {
|
|
7427
7566
|
bypassRecentDedupe: true,
|
|
7428
7567
|
bypassKnownPeerHints: mode === "churn" ||
|
|
@@ -7473,6 +7612,10 @@ let SharedLog = (() => {
|
|
|
7473
7612
|
if ((this._nativeBackbone ?? this._nativeSharedLogState) &&
|
|
7474
7613
|
residentEntriesByHash &&
|
|
7475
7614
|
!this.hasCustomFindLeaders()) {
|
|
7615
|
+
if (profileCounts) {
|
|
7616
|
+
profileCounts.nativePasses += 1;
|
|
7617
|
+
profileCounts.inputEntries += residentEntriesByHash.size;
|
|
7618
|
+
}
|
|
7476
7619
|
const repairDispatchPlan = pruneStaleJoinWarmupPeers()
|
|
7477
7620
|
? await this.planResidentRepairDispatchBatch({
|
|
7478
7621
|
pendingModes,
|
|
@@ -7505,6 +7648,8 @@ let SharedLog = (() => {
|
|
|
7505
7648
|
!iterator.done() &&
|
|
7506
7649
|
pruneStaleJoinWarmupPeers()) {
|
|
7507
7650
|
const entries = await iterator.next(REPAIR_SWEEP_ENTRY_BATCH_SIZE);
|
|
7651
|
+
if (profileCounts)
|
|
7652
|
+
profileCounts.inputEntries += entries.length;
|
|
7508
7653
|
if (!this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
7509
7654
|
return;
|
|
7510
7655
|
}
|
|
@@ -7619,6 +7764,8 @@ let SharedLog = (() => {
|
|
|
7619
7764
|
}
|
|
7620
7765
|
}
|
|
7621
7766
|
catch (error) {
|
|
7767
|
+
if (profileCounts)
|
|
7768
|
+
profileCounts.outcome = "error";
|
|
7622
7769
|
if (this.isRepairLifecycleActive(repairLifecycleController) &&
|
|
7623
7770
|
!isNotStartedError(error)) {
|
|
7624
7771
|
logger.error(`Repair sweep failed: ${error?.message ?? error}`);
|
|
@@ -7635,6 +7782,21 @@ let SharedLog = (() => {
|
|
|
7635
7782
|
void this.runRepairSweep(repairLifecycleController);
|
|
7636
7783
|
}
|
|
7637
7784
|
}
|
|
7785
|
+
if (profileCounts) {
|
|
7786
|
+
emitAdvisorySyncProfileDuration(profile, startedAt, {
|
|
7787
|
+
name: "sharedLog.placement.pass",
|
|
7788
|
+
component: "shared-log",
|
|
7789
|
+
entries: profileCounts.inputEntries,
|
|
7790
|
+
count: profileCounts.repairCandidates,
|
|
7791
|
+
details: {
|
|
7792
|
+
phase: "repair-sweep",
|
|
7793
|
+
outcome: profileCounts.outcome,
|
|
7794
|
+
passes: profileCounts.passes,
|
|
7795
|
+
nativePasses: profileCounts.nativePasses,
|
|
7796
|
+
repairBatches: profileCounts.repairBatches,
|
|
7797
|
+
},
|
|
7798
|
+
});
|
|
7799
|
+
}
|
|
7638
7800
|
}
|
|
7639
7801
|
}
|
|
7640
7802
|
async pruneDebouncedFnAddIfNotKeeping(args, ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(), additionalCurrentCheck) {
|
|
@@ -14013,6 +14175,10 @@ let SharedLog = (() => {
|
|
|
14013
14175
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
14014
14176
|
try {
|
|
14015
14177
|
this.stopSubscriptionChangeCallbackAdmission();
|
|
14178
|
+
// A receive may be awaiting a synchronizer response shipment. Cancel
|
|
14179
|
+
// its dispatch generation before draining that receive, rather than
|
|
14180
|
+
// waiting for _close() to reach the synchronizer's final teardown.
|
|
14181
|
+
this.syncronizer?.beginClose?.();
|
|
14016
14182
|
this.joinWarmup.cancelAllJoinWarmupTargets();
|
|
14017
14183
|
await this.drainSubscriptionChangeCallbacks();
|
|
14018
14184
|
// An already-admitted subscription callback can create a fresh warmup
|
|
@@ -14149,6 +14315,7 @@ let SharedLog = (() => {
|
|
|
14149
14315
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
14150
14316
|
try {
|
|
14151
14317
|
this.stopSubscriptionChangeCallbackAdmission();
|
|
14318
|
+
this.syncronizer?.beginClose?.();
|
|
14152
14319
|
this.joinWarmup.cancelAllJoinWarmupTargets();
|
|
14153
14320
|
await this.drainSubscriptionChangeCallbacks();
|
|
14154
14321
|
// An already-admitted subscription callback can create a fresh warmup
|
|
@@ -14391,10 +14558,11 @@ let SharedLog = (() => {
|
|
|
14391
14558
|
const rawExistingStartedAt = syncProfileStart(syncProfile);
|
|
14392
14559
|
const rawExistingHashes = await this.log.hasMany(msg.heads.map((head) => head.hash));
|
|
14393
14560
|
if (syncProfile) {
|
|
14394
|
-
|
|
14561
|
+
emitAdvisorySyncProfileDuration(syncProfile, rawExistingStartedAt, {
|
|
14395
14562
|
name: "sharedLog.rawReceive.existingHeads",
|
|
14396
14563
|
component: "shared-log",
|
|
14397
14564
|
entries: msg.heads.length,
|
|
14565
|
+
count: rawExistingHashes.size,
|
|
14398
14566
|
messages: 1,
|
|
14399
14567
|
});
|
|
14400
14568
|
}
|
|
@@ -14711,10 +14879,11 @@ let SharedLog = (() => {
|
|
|
14711
14879
|
? undefined
|
|
14712
14880
|
: await this.log.hasMany(headHashes);
|
|
14713
14881
|
if (syncProfile) {
|
|
14714
|
-
|
|
14882
|
+
emitAdvisorySyncProfileDuration(syncProfile, existingStartedAt, {
|
|
14715
14883
|
name: "sharedLog.receive.existingHeads",
|
|
14716
14884
|
component: "shared-log",
|
|
14717
14885
|
entries: heads.length,
|
|
14886
|
+
count: existingHashes?.size,
|
|
14718
14887
|
messages: 1,
|
|
14719
14888
|
details: { rawMaterializedKnownMissing },
|
|
14720
14889
|
});
|
|
@@ -20429,6 +20598,17 @@ let SharedLog = (() => {
|
|
|
20429
20598
|
warmupSessions.get(target);
|
|
20430
20599
|
const areJoinWarmupGenerationsCurrent = () => isOwnershipLifecycleCurrent() &&
|
|
20431
20600
|
[...warmupPeers].every(isCurrentJoinWarmupTarget);
|
|
20601
|
+
const profile = this._logProperties?.sync?.profile;
|
|
20602
|
+
const profileStartedAt = syncProfileStart(profile);
|
|
20603
|
+
const profileCounts = profile
|
|
20604
|
+
? {
|
|
20605
|
+
examinedEntries: 0,
|
|
20606
|
+
repairCandidates: 0,
|
|
20607
|
+
repairBatches: 0,
|
|
20608
|
+
pruneScan: false,
|
|
20609
|
+
outcome: "stale",
|
|
20610
|
+
}
|
|
20611
|
+
: undefined;
|
|
20432
20612
|
try {
|
|
20433
20613
|
const uncheckedDeliver = new Map();
|
|
20434
20614
|
const flushUncheckedDeliverTarget = (target) => {
|
|
@@ -20449,6 +20629,10 @@ let SharedLog = (() => {
|
|
|
20449
20629
|
: isWarmupTarget
|
|
20450
20630
|
? "join-warmup"
|
|
20451
20631
|
: "join-authoritative";
|
|
20632
|
+
if (profileCounts) {
|
|
20633
|
+
profileCounts.repairCandidates += entries.size;
|
|
20634
|
+
profileCounts.repairBatches += 1;
|
|
20635
|
+
}
|
|
20452
20636
|
this.dispatchMaybeMissingEntries(target, entries, {
|
|
20453
20637
|
bypassRecentDedupe: isWarmupTarget || forceFreshDelivery,
|
|
20454
20638
|
bypassKnownPeerHints: forceFreshDelivery ||
|
|
@@ -20487,6 +20671,8 @@ let SharedLog = (() => {
|
|
|
20487
20671
|
for await (const entryReplicated of toRebalance(immediateRebalanceChanges, this.entryCoordinatesIndex, this.recentlyRebalanced, {
|
|
20488
20672
|
forceFresh: forceFreshDelivery || useJoinWarmupFastPath,
|
|
20489
20673
|
})) {
|
|
20674
|
+
if (profileCounts)
|
|
20675
|
+
profileCounts.examinedEntries += 1;
|
|
20490
20676
|
if (!isOwnershipLifecycleCurrent() ||
|
|
20491
20677
|
(useJoinWarmupFastPath && !areJoinWarmupGenerationsCurrent())) {
|
|
20492
20678
|
break;
|
|
@@ -20683,6 +20869,8 @@ let SharedLog = (() => {
|
|
|
20683
20869
|
change.type === "removed" ||
|
|
20684
20870
|
change.type === "replaced"));
|
|
20685
20871
|
if (shouldRunLocalPruneScan) {
|
|
20872
|
+
if (profileCounts)
|
|
20873
|
+
profileCounts.pruneScan = true;
|
|
20686
20874
|
throwIfOwnershipLifecycleInactive();
|
|
20687
20875
|
// Adaptive range changes and fixed zero-width updates can make already-indexed
|
|
20688
20876
|
// local heads prunable even when the incremental rebalance scan misses them
|
|
@@ -20701,6 +20889,8 @@ let SharedLog = (() => {
|
|
|
20701
20889
|
return false;
|
|
20702
20890
|
}
|
|
20703
20891
|
}
|
|
20892
|
+
if (profileCounts)
|
|
20893
|
+
profileCounts.outcome = "completed";
|
|
20704
20894
|
return changed;
|
|
20705
20895
|
}
|
|
20706
20896
|
catch (error) {
|
|
@@ -20710,9 +20900,30 @@ let SharedLog = (() => {
|
|
|
20710
20900
|
if (isNotStartedError(error)) {
|
|
20711
20901
|
return false; // we are not started yet, so no changes
|
|
20712
20902
|
}
|
|
20903
|
+
if (profileCounts)
|
|
20904
|
+
profileCounts.outcome = "error";
|
|
20713
20905
|
logger.error(error.toString());
|
|
20714
20906
|
throw error;
|
|
20715
20907
|
}
|
|
20908
|
+
finally {
|
|
20909
|
+
if (profileCounts) {
|
|
20910
|
+
emitAdvisorySyncProfileDuration(profile, profileStartedAt, {
|
|
20911
|
+
name: "sharedLog.placement.pass",
|
|
20912
|
+
component: "shared-log",
|
|
20913
|
+
entries: profileCounts.examinedEntries,
|
|
20914
|
+
count: profileCounts.repairCandidates,
|
|
20915
|
+
details: {
|
|
20916
|
+
phase: "range-change",
|
|
20917
|
+
outcome: profileCounts.outcome,
|
|
20918
|
+
changes: changes.length,
|
|
20919
|
+
repairBatches: profileCounts.repairBatches,
|
|
20920
|
+
pruneScan: profileCounts.pruneScan,
|
|
20921
|
+
forceFreshDelivery,
|
|
20922
|
+
joinWarmupFastPath: useJoinWarmupFastPath,
|
|
20923
|
+
},
|
|
20924
|
+
});
|
|
20925
|
+
}
|
|
20926
|
+
}
|
|
20716
20927
|
}
|
|
20717
20928
|
async _onUnsubscription(evt) {
|
|
20718
20929
|
logger.trace(`Peer disconnected '${evt.detail.from.hashcode()}' from '${JSON.stringify(evt.detail.topics.map((x) => x))} '`);
|
|
@@ -20741,6 +20952,13 @@ let SharedLog = (() => {
|
|
|
20741
20952
|
await this.handleSubscriptionChange(evt.detail.from, evt.detail.topics, true, subscriptionEpoch, evt.detail.session);
|
|
20742
20953
|
}
|
|
20743
20954
|
async rebalanceParticipation(ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(), rebalanceParticipationDebounced = this.rebalanceParticipationDebounced) {
|
|
20955
|
+
const profile = this._isAdaptiveReplicating
|
|
20956
|
+
? this._logProperties?.sync?.profile
|
|
20957
|
+
: undefined;
|
|
20958
|
+
const profileStartedAt = syncProfileStart(profile);
|
|
20959
|
+
const profileDetails = profile
|
|
20960
|
+
? { outcome: "stale", idleRemainingMs: 0 }
|
|
20961
|
+
: undefined;
|
|
20744
20962
|
// Stage 3: the lifecycle owns all three identity terms. `lifecycle` may
|
|
20745
20963
|
// go stale later; its deps late-bind to the host, so the debouncer term
|
|
20746
20964
|
// still reads the current host field, and the role term can disagree
|
|
@@ -20772,6 +20990,11 @@ let SharedLog = (() => {
|
|
|
20772
20990
|
}
|
|
20773
20991
|
if (this._isAdaptiveReplicating) {
|
|
20774
20992
|
if (this.shouldDelayAdaptiveRebalance()) {
|
|
20993
|
+
if (profileDetails) {
|
|
20994
|
+
profileDetails.outcome = "idle-deferred";
|
|
20995
|
+
profileDetails.idleRemainingMs = Math.max(0, this.adaptiveRebalanceIdleMs -
|
|
20996
|
+
(Date.now() - this._lastLocalAppendAt));
|
|
20997
|
+
}
|
|
20775
20998
|
if (isCurrent()) {
|
|
20776
20999
|
void rebalanceParticipationDebounced?.call();
|
|
20777
21000
|
}
|
|
@@ -20781,11 +21004,18 @@ let SharedLog = (() => {
|
|
|
20781
21004
|
const usedMemory = await this.getMemoryUsage();
|
|
20782
21005
|
if (!isCurrent())
|
|
20783
21006
|
return false;
|
|
21007
|
+
if (profileDetails) {
|
|
21008
|
+
profileDetails.storageUsedBytes = usedMemory;
|
|
21009
|
+
profileDetails.storageObjectiveBytes =
|
|
21010
|
+
this.replicationController.maxMemoryLimit;
|
|
21011
|
+
}
|
|
20784
21012
|
this.scheduleReplicationStatusRefreshForStorage(usedMemory);
|
|
20785
21013
|
let dynamicRange = await this.getDynamicRange();
|
|
20786
21014
|
if (!isCurrent())
|
|
20787
21015
|
return false;
|
|
20788
21016
|
if (!dynamicRange) {
|
|
21017
|
+
if (profileDetails)
|
|
21018
|
+
profileDetails.outcome = "not-permitted";
|
|
20789
21019
|
return; // not allowed to replicate
|
|
20790
21020
|
}
|
|
20791
21021
|
if (this.replicationController.maxMemoryLimit != null &&
|
|
@@ -20805,13 +21035,24 @@ let SharedLog = (() => {
|
|
|
20805
21035
|
const totalParticipation = await this.calculateTotalParticipation();
|
|
20806
21036
|
if (!isCurrent())
|
|
20807
21037
|
return false;
|
|
21038
|
+
const cpuUsage = this.cpuUsage?.value();
|
|
21039
|
+
const stepStartedAt = syncProfileStart(profile);
|
|
20808
21040
|
const newFactor = this.replicationController.step({
|
|
20809
21041
|
memoryUsage: usedMemory,
|
|
20810
21042
|
currentFactor: dynamicRange.widthNormalized,
|
|
20811
21043
|
totalFactor: totalParticipation, // TODO use this._totalParticipation when flakiness is fixed
|
|
20812
21044
|
peerCount: peersSize,
|
|
20813
|
-
cpuUsage
|
|
21045
|
+
cpuUsage,
|
|
20814
21046
|
});
|
|
21047
|
+
if (profileDetails) {
|
|
21048
|
+
profileDetails.preStepMs = stepStartedAt - profileStartedAt;
|
|
21049
|
+
profileDetails.stepMs = syncProfileStart(profile) - stepStartedAt;
|
|
21050
|
+
profileDetails.currentFactor = dynamicRange.widthNormalized;
|
|
21051
|
+
profileDetails.proposedFactor = newFactor;
|
|
21052
|
+
profileDetails.totalFactor = totalParticipation;
|
|
21053
|
+
profileDetails.controllerPeerCount = peersSize;
|
|
21054
|
+
profileDetails.cpuUsage = cpuUsage;
|
|
21055
|
+
}
|
|
20815
21056
|
const absoluteDifference = Math.abs(dynamicRange.widthNormalized - newFactor);
|
|
20816
21057
|
const relativeDifference = absoluteDifference /
|
|
20817
21058
|
Math.max(dynamicRange.widthNormalized, RECALCULATE_PARTICIPATION_RELATIVE_DENOMINATOR_FLOOR);
|
|
@@ -20839,8 +21080,11 @@ let SharedLog = (() => {
|
|
|
20839
21080
|
if (!isCurrent())
|
|
20840
21081
|
return false;
|
|
20841
21082
|
if (!canReplicate) {
|
|
21083
|
+
if (profileDetails)
|
|
21084
|
+
profileDetails.outcome = "not-permitted";
|
|
20842
21085
|
return false;
|
|
20843
21086
|
}
|
|
21087
|
+
const applyStartedAt = syncProfileStart(profile);
|
|
20844
21088
|
await this.startAnnounceReplicating([dynamicRange], {
|
|
20845
21089
|
checkDuplicates: false,
|
|
20846
21090
|
reset: false,
|
|
@@ -20848,6 +21092,10 @@ let SharedLog = (() => {
|
|
|
20848
21092
|
}, ownershipLifecycleController);
|
|
20849
21093
|
if (!isCurrent())
|
|
20850
21094
|
return false;
|
|
21095
|
+
if (profileDetails) {
|
|
21096
|
+
profileDetails.outcome = "apply-settled";
|
|
21097
|
+
profileDetails.applyMs = syncProfileStart(profile) - applyStartedAt;
|
|
21098
|
+
}
|
|
20851
21099
|
/* await this._updateRole(newRole, onRoleChange); */
|
|
20852
21100
|
if (isCurrent()) {
|
|
20853
21101
|
void rebalanceParticipationDebounced?.call();
|
|
@@ -20855,6 +21103,8 @@ let SharedLog = (() => {
|
|
|
20855
21103
|
return true;
|
|
20856
21104
|
}
|
|
20857
21105
|
else {
|
|
21106
|
+
if (profileDetails)
|
|
21107
|
+
profileDetails.outcome = "unchanged";
|
|
20858
21108
|
if (isCurrent()) {
|
|
20859
21109
|
void rebalanceParticipationDebounced?.call();
|
|
20860
21110
|
}
|
|
@@ -20863,13 +21113,27 @@ let SharedLog = (() => {
|
|
|
20863
21113
|
}
|
|
20864
21114
|
return false;
|
|
20865
21115
|
};
|
|
20866
|
-
|
|
20867
|
-
|
|
20868
|
-
|
|
21116
|
+
try {
|
|
21117
|
+
return await fn().catch((error) => {
|
|
21118
|
+
if (isNotStartedError(error) || isClosedStoreRace(error)) {
|
|
21119
|
+
if (profileDetails)
|
|
21120
|
+
profileDetails.outcome = "stale";
|
|
21121
|
+
return false;
|
|
21122
|
+
}
|
|
21123
|
+
if (profileDetails)
|
|
21124
|
+
profileDetails.outcome = "error";
|
|
21125
|
+
throw error;
|
|
21126
|
+
});
|
|
21127
|
+
}
|
|
21128
|
+
finally {
|
|
21129
|
+
if (profileDetails) {
|
|
21130
|
+
emitAdvisorySyncProfileDuration(profile, profileStartedAt, {
|
|
21131
|
+
name: "sharedLog.adaptive.rebalance",
|
|
21132
|
+
component: "shared-log",
|
|
21133
|
+
details: profileDetails,
|
|
21134
|
+
});
|
|
20869
21135
|
}
|
|
20870
|
-
|
|
20871
|
-
});
|
|
20872
|
-
return resp;
|
|
21136
|
+
}
|
|
20873
21137
|
}
|
|
20874
21138
|
getDynamicRangeOffset() {
|
|
20875
21139
|
const options = this._logProperties
|