@peerbit/shared-log 16.0.20 → 16.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +218 -21
- package/dist/src/index.js.map +1 -1
- package/dist/src/replication-info-v2-receive.d.ts +23 -0
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -1
- package/dist/src/replication-info-v2-receive.js +44 -10
- package/dist/src/replication-info-v2-receive.js.map +1 -1
- package/dist/src/replication-info-v2-send.d.ts +15 -0
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +89 -30
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/package.json +13 -13
- package/src/index.ts +314 -20
- package/src/replication-info-v2-receive.ts +66 -14
- package/src/replication-info-v2-send.ts +150 -38
package/src/index.ts
CHANGED
|
@@ -719,6 +719,14 @@ type PersistedDeliveryPlanningRecord<T, R extends "u32" | "u64"> = Readonly<{
|
|
|
719
719
|
createFullPlanningSource?: () => Entry<T>;
|
|
720
720
|
}>;
|
|
721
721
|
|
|
722
|
+
type PersistedAppendBackfillSource<T, R extends "u32" | "u64"> = {
|
|
723
|
+
entry: Entry<T>;
|
|
724
|
+
coordinates: NumberFromType<R>[];
|
|
725
|
+
assignmentExtraLeaders: LeaderMap;
|
|
726
|
+
deliveryExtraTargets: Set<string>;
|
|
727
|
+
extrasOwnershipRevision?: number;
|
|
728
|
+
};
|
|
729
|
+
|
|
722
730
|
type NativeBackboneSimpleDocumentProjectionPlan = {
|
|
723
731
|
documentVariantType?: "u8" | "string";
|
|
724
732
|
documentVariantValue?: string;
|
|
@@ -5475,6 +5483,10 @@ export class SharedLog<
|
|
|
5475
5483
|
): { capabilitySession: bigint; peerSession: PeerSession } | undefined {
|
|
5476
5484
|
const capabilitySession = this._peerSyncCapabilitySessions.get(peerHash);
|
|
5477
5485
|
const peerSession = this._peerSessions.current(peerHash);
|
|
5486
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
5487
|
+
const requiredCapabilities =
|
|
5488
|
+
SYNC_CAPABILITY_PERSISTED_ENTRY_RECEIPTS |
|
|
5489
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_CONFIRM;
|
|
5478
5490
|
if (
|
|
5479
5491
|
capabilitySession == null ||
|
|
5480
5492
|
!peerSession ||
|
|
@@ -5482,8 +5494,14 @@ export class SharedLog<
|
|
|
5482
5494
|
!this._peerSessions.isCurrent(peerHash, peerSession) ||
|
|
5483
5495
|
!this._peerSyncCapabilityTimestamps.has(peerHash) ||
|
|
5484
5496
|
((this._peerSyncCapabilities.get(peerHash) ?? 0) &
|
|
5485
|
-
|
|
5486
|
-
|
|
5497
|
+
requiredCapabilities) !==
|
|
5498
|
+
requiredCapabilities ||
|
|
5499
|
+
!this._v2Receive.isCurrentActive({
|
|
5500
|
+
peerHash,
|
|
5501
|
+
peerSession,
|
|
5502
|
+
receiveEpoch,
|
|
5503
|
+
senderTransportSession: capabilitySession,
|
|
5504
|
+
})
|
|
5487
5505
|
) {
|
|
5488
5506
|
return undefined;
|
|
5489
5507
|
}
|
|
@@ -5754,6 +5772,10 @@ export class SharedLog<
|
|
|
5754
5772
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
5755
5773
|
persistedDeadline?: PersistedDeliveryDeadline,
|
|
5756
5774
|
transferOnFirstRound = false,
|
|
5775
|
+
onFreshLeaderPlan?: (
|
|
5776
|
+
leadersByEntry: readonly LeaderMap[],
|
|
5777
|
+
ownershipRevision: number,
|
|
5778
|
+
) => void,
|
|
5757
5779
|
): Promise<void> {
|
|
5758
5780
|
const minAcks = Math.floor(delivery.minAcks!);
|
|
5759
5781
|
const records = new Map(
|
|
@@ -5844,6 +5866,8 @@ export class SharedLog<
|
|
|
5844
5866
|
ownershipLifecycleController,
|
|
5845
5867
|
);
|
|
5846
5868
|
if (!isRoundOwnershipCurrent()) continue;
|
|
5869
|
+
onFreshLeaderPlan?.(leadersByEntry, ownershipRevision);
|
|
5870
|
+
if (!isRoundOwnershipCurrent()) continue;
|
|
5847
5871
|
const selfHash = this.node.identity.publicKey.hashcode();
|
|
5848
5872
|
if (needsInitialLeaderCheck) {
|
|
5849
5873
|
needsInitialLeaderCheck = false;
|
|
@@ -5946,6 +5970,32 @@ export class SharedLog<
|
|
|
5946
5970
|
current.peerSession === captured.peerSession
|
|
5947
5971
|
);
|
|
5948
5972
|
};
|
|
5973
|
+
try {
|
|
5974
|
+
await this._v2Send.confirmLatestForPeer(
|
|
5975
|
+
{
|
|
5976
|
+
peerHash: peer,
|
|
5977
|
+
peerSession: captured.peerSession,
|
|
5978
|
+
receiverTransportSession: captured.capabilitySession,
|
|
5979
|
+
},
|
|
5980
|
+
{
|
|
5981
|
+
timeout: getAttemptTimeout(),
|
|
5982
|
+
signal: roundSignal,
|
|
5983
|
+
},
|
|
5984
|
+
);
|
|
5985
|
+
} catch {
|
|
5986
|
+
// The exact receiver generation did not prove that it applied
|
|
5987
|
+
// our latest role state. Replan instead of transferring to a
|
|
5988
|
+
// peer that can still make a stale admission decision.
|
|
5989
|
+
if (transferAllOnRound && isPeerRoundCurrent()) {
|
|
5990
|
+
const state = ensureRepairState();
|
|
5991
|
+
for (const hash of hashes) state.hashes.add(hash);
|
|
5992
|
+
}
|
|
5993
|
+
return;
|
|
5994
|
+
}
|
|
5995
|
+
if (!isPeerRoundCurrent()) {
|
|
5996
|
+
purgePeerDeliveryState(peer);
|
|
5997
|
+
return;
|
|
5998
|
+
}
|
|
5949
5999
|
const repairs = repairsByPeer.get(peer)?.hashes;
|
|
5950
6000
|
const transferHashes = transferAllOnRound
|
|
5951
6001
|
? hashes
|
|
@@ -6122,6 +6172,9 @@ export class SharedLog<
|
|
|
6122
6172
|
replicas,
|
|
6123
6173
|
ownershipLifecycleController,
|
|
6124
6174
|
);
|
|
6175
|
+
if (!isRoundOwnershipCurrent()) return false;
|
|
6176
|
+
onFreshLeaderPlan?.(validatedLeaders, ownershipRevision);
|
|
6177
|
+
if (!isRoundOwnershipCurrent()) return false;
|
|
6125
6178
|
for (let index = 0; index < entryArray.length; index++) {
|
|
6126
6179
|
if (signal.aborted || !isRoundOwnershipCurrent()) {
|
|
6127
6180
|
if (signal.aborted) {
|
|
@@ -6205,6 +6258,75 @@ export class SharedLog<
|
|
|
6205
6258
|
}
|
|
6206
6259
|
}
|
|
6207
6260
|
|
|
6261
|
+
private async collectDeferredAppendBackfillExtras(
|
|
6262
|
+
entry: Entry<T>,
|
|
6263
|
+
replicas: number,
|
|
6264
|
+
baseLeaders: LeaderMap,
|
|
6265
|
+
nativeDeliveryPlan: AppendDeliveryPlan | undefined,
|
|
6266
|
+
ownershipLifecycleController: AbortController,
|
|
6267
|
+
): Promise<
|
|
6268
|
+
Omit<PersistedAppendBackfillSource<T, R>, "entry" | "coordinates">
|
|
6269
|
+
> {
|
|
6270
|
+
const ownershipRevision =
|
|
6271
|
+
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
6272
|
+
const ownershipWasStable =
|
|
6273
|
+
this.isReceiveOwnershipSnapshotStable(ownershipRevision);
|
|
6274
|
+
const assignmentExtraLeaders: LeaderMap = new Map();
|
|
6275
|
+
const deliveryExtraTargets = new Set<string>();
|
|
6276
|
+
if (nativeDeliveryPlan) {
|
|
6277
|
+
for (const peer of nativeDeliveryPlan.repairTargets) {
|
|
6278
|
+
// The sampled entry leaders are replaced by settlement's fresh plan.
|
|
6279
|
+
// Retain only native repair additions that were outside that base map.
|
|
6280
|
+
if (!baseLeaders.has(peer)) deliveryExtraTargets.add(peer);
|
|
6281
|
+
}
|
|
6282
|
+
} else {
|
|
6283
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
6284
|
+
const fullReplicaDeliveryCandidates =
|
|
6285
|
+
await this.getNativeFullReplicaDeliveryCandidates(replicas, selfHash);
|
|
6286
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
6287
|
+
ownershipLifecycleController,
|
|
6288
|
+
);
|
|
6289
|
+
if (replicas >= Math.max(1, fullReplicaDeliveryCandidates.size)) {
|
|
6290
|
+
for (const peer of fullReplicaDeliveryCandidates) {
|
|
6291
|
+
if (!baseLeaders.has(peer)) {
|
|
6292
|
+
assignmentExtraLeaders.set(peer, { intersecting: true });
|
|
6293
|
+
}
|
|
6294
|
+
}
|
|
6295
|
+
}
|
|
6296
|
+
}
|
|
6297
|
+
|
|
6298
|
+
const referenceLeaders: LeaderMap = new Map();
|
|
6299
|
+
for await (const message of createExchangeHeadsMessages(this.log, [
|
|
6300
|
+
entry,
|
|
6301
|
+
])) {
|
|
6302
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
6303
|
+
ownershipLifecycleController,
|
|
6304
|
+
);
|
|
6305
|
+
await this._mergeLeadersFromGidReferences(
|
|
6306
|
+
message,
|
|
6307
|
+
replicas,
|
|
6308
|
+
referenceLeaders,
|
|
6309
|
+
ownershipLifecycleController,
|
|
6310
|
+
{ freshLeaderPlan: true },
|
|
6311
|
+
);
|
|
6312
|
+
}
|
|
6313
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
6314
|
+
ownershipLifecycleController,
|
|
6315
|
+
);
|
|
6316
|
+
for (const peer of referenceLeaders.keys()) {
|
|
6317
|
+
deliveryExtraTargets.add(peer);
|
|
6318
|
+
}
|
|
6319
|
+
return {
|
|
6320
|
+
assignmentExtraLeaders,
|
|
6321
|
+
deliveryExtraTargets,
|
|
6322
|
+
extrasOwnershipRevision:
|
|
6323
|
+
ownershipWasStable &&
|
|
6324
|
+
this.isReceiveOwnershipSnapshotStable(ownershipRevision)
|
|
6325
|
+
? ownershipRevision
|
|
6326
|
+
: undefined,
|
|
6327
|
+
};
|
|
6328
|
+
}
|
|
6329
|
+
|
|
6208
6330
|
private async _appendDeliverToReplicators(
|
|
6209
6331
|
entry: Entry<T>,
|
|
6210
6332
|
coordinates: NumberFromType<R>[],
|
|
@@ -6299,7 +6421,11 @@ export class SharedLog<
|
|
|
6299
6421
|
if (!delivery) {
|
|
6300
6422
|
for (const peer of nativeDeliveryPlan.repairTargets) {
|
|
6301
6423
|
throwIfInactive();
|
|
6302
|
-
this.queueAppendBackfill(
|
|
6424
|
+
this.queueAppendBackfill(
|
|
6425
|
+
peer,
|
|
6426
|
+
entryReplicatedForRepair,
|
|
6427
|
+
ownershipLifecycleController,
|
|
6428
|
+
);
|
|
6303
6429
|
}
|
|
6304
6430
|
if (nativeDeliveryPlan.defaultSendSilent) {
|
|
6305
6431
|
const rawTargets = this.canUseLiveRawGossip(
|
|
@@ -6372,7 +6498,11 @@ export class SharedLog<
|
|
|
6372
6498
|
}
|
|
6373
6499
|
for (const peer of nativeDeliveryPlan.repairTargets) {
|
|
6374
6500
|
throwIfInactive();
|
|
6375
|
-
this.queueAppendBackfill(
|
|
6501
|
+
this.queueAppendBackfill(
|
|
6502
|
+
peer,
|
|
6503
|
+
entryReplicatedForRepair,
|
|
6504
|
+
ownershipLifecycleController,
|
|
6505
|
+
);
|
|
6376
6506
|
}
|
|
6377
6507
|
continue;
|
|
6378
6508
|
}
|
|
@@ -6427,7 +6557,11 @@ export class SharedLog<
|
|
|
6427
6557
|
// delivery acks, we still need a targeted backfill source of truth for the
|
|
6428
6558
|
// authoritative recipients or one entry can get stuck at 2/3 replicas
|
|
6429
6559
|
// forever. Best-effort fallback subscribers are not repair-worthy.
|
|
6430
|
-
this.queueAppendBackfill(
|
|
6560
|
+
this.queueAppendBackfill(
|
|
6561
|
+
peer,
|
|
6562
|
+
entryReplicatedForRepair,
|
|
6563
|
+
ownershipLifecycleController,
|
|
6564
|
+
);
|
|
6431
6565
|
}
|
|
6432
6566
|
if (isLeader) {
|
|
6433
6567
|
const rawTargets = this.canUseLiveRawGossip(set, selfHash);
|
|
@@ -6536,7 +6670,11 @@ export class SharedLog<
|
|
|
6536
6670
|
// Direct append delivery is intentionally optimistic. Queue one delayed,
|
|
6537
6671
|
// batched maybe-sync pass for the intended recipients so stable 3-peer
|
|
6538
6672
|
// append workloads do not depend on perfect first-try delivery ordering.
|
|
6539
|
-
this.queueAppendBackfill(
|
|
6673
|
+
this.queueAppendBackfill(
|
|
6674
|
+
peer,
|
|
6675
|
+
entryReplicatedForRepair,
|
|
6676
|
+
ownershipLifecycleController,
|
|
6677
|
+
);
|
|
6540
6678
|
}
|
|
6541
6679
|
}
|
|
6542
6680
|
|
|
@@ -6551,6 +6689,7 @@ export class SharedLog<
|
|
|
6551
6689
|
minReplicasValue: number,
|
|
6552
6690
|
leaders: LeaderMap,
|
|
6553
6691
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
6692
|
+
options?: { freshLeaderPlan?: boolean },
|
|
6554
6693
|
) {
|
|
6555
6694
|
const throwIfInactive = () =>
|
|
6556
6695
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
@@ -6576,13 +6715,13 @@ export class SharedLog<
|
|
|
6576
6715
|
found = await this.findLeadersFromEntry(
|
|
6577
6716
|
gidEntry,
|
|
6578
6717
|
minReplicasValue,
|
|
6579
|
-
undefined,
|
|
6718
|
+
options?.freshLeaderPlan ? { freshLeaderPlan: true } : undefined,
|
|
6580
6719
|
ownershipLifecycleController,
|
|
6581
6720
|
);
|
|
6582
6721
|
} else {
|
|
6583
6722
|
found = await this._findLeaders(
|
|
6584
6723
|
coordinates,
|
|
6585
|
-
undefined,
|
|
6724
|
+
options?.freshLeaderPlan ? { freshLeaderPlan: true } : undefined,
|
|
6586
6725
|
ownershipLifecycleController,
|
|
6587
6726
|
);
|
|
6588
6727
|
}
|
|
@@ -6891,8 +7030,13 @@ export class SharedLog<
|
|
|
6891
7030
|
private canCacheLeaderSelectionContext(options?: {
|
|
6892
7031
|
roleAge?: number;
|
|
6893
7032
|
candidates?: Iterable<string>;
|
|
7033
|
+
freshLeaderPlan?: boolean;
|
|
6894
7034
|
}) {
|
|
6895
|
-
return
|
|
7035
|
+
return (
|
|
7036
|
+
options?.roleAge == null &&
|
|
7037
|
+
options?.candidates == null &&
|
|
7038
|
+
options?.freshLeaderPlan !== true
|
|
7039
|
+
);
|
|
6896
7040
|
}
|
|
6897
7041
|
|
|
6898
7042
|
private cloneLeaderSelectionContext(
|
|
@@ -6910,6 +7054,7 @@ export class SharedLog<
|
|
|
6910
7054
|
private getCachedLeaderSelectionContext(options?: {
|
|
6911
7055
|
roleAge?: number;
|
|
6912
7056
|
candidates?: Iterable<string>;
|
|
7057
|
+
freshLeaderPlan?: boolean;
|
|
6913
7058
|
}): LeaderSelectionContext | undefined {
|
|
6914
7059
|
if (!this.canCacheLeaderSelectionContext(options)) {
|
|
6915
7060
|
return;
|
|
@@ -6926,6 +7071,7 @@ export class SharedLog<
|
|
|
6926
7071
|
| {
|
|
6927
7072
|
roleAge?: number;
|
|
6928
7073
|
candidates?: Iterable<string>;
|
|
7074
|
+
freshLeaderPlan?: boolean;
|
|
6929
7075
|
}
|
|
6930
7076
|
| undefined,
|
|
6931
7077
|
context: LeaderSelectionContext,
|
|
@@ -10191,10 +10337,7 @@ export class SharedLog<
|
|
|
10191
10337
|
});
|
|
10192
10338
|
}
|
|
10193
10339
|
|
|
10194
|
-
private flushAppendBackfill(
|
|
10195
|
-
repairLifecycleController: AbortController = this._instanceLifecycle
|
|
10196
|
-
?.ownershipLifecycleController as AbortController,
|
|
10197
|
-
) {
|
|
10340
|
+
private flushAppendBackfill(repairLifecycleController: AbortController) {
|
|
10198
10341
|
if (
|
|
10199
10342
|
!this.isRepairLifecycleActive(repairLifecycleController) ||
|
|
10200
10343
|
this._appendBackfillPendingByTarget.size === 0
|
|
@@ -10215,9 +10358,11 @@ export class SharedLog<
|
|
|
10215
10358
|
}
|
|
10216
10359
|
}
|
|
10217
10360
|
|
|
10218
|
-
private queueAppendBackfill(
|
|
10219
|
-
|
|
10220
|
-
|
|
10361
|
+
private queueAppendBackfill(
|
|
10362
|
+
target: string,
|
|
10363
|
+
entry: EntryReplicated<R>,
|
|
10364
|
+
repairLifecycleController: AbortController,
|
|
10365
|
+
) {
|
|
10221
10366
|
if (!this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
10222
10367
|
return;
|
|
10223
10368
|
}
|
|
@@ -10249,6 +10394,62 @@ export class SharedLog<
|
|
|
10249
10394
|
this._appendBackfillTimer = timer;
|
|
10250
10395
|
}
|
|
10251
10396
|
|
|
10397
|
+
private queuePersistedAppendBackfill(
|
|
10398
|
+
source: PersistedAppendBackfillSource<T, R>,
|
|
10399
|
+
leaders: LeaderMap,
|
|
10400
|
+
replicas: number,
|
|
10401
|
+
repairLifecycleController: AbortController,
|
|
10402
|
+
ownershipRevision: number,
|
|
10403
|
+
): void {
|
|
10404
|
+
try {
|
|
10405
|
+
if (
|
|
10406
|
+
!this.isRepairLifecycleActive(repairLifecycleController) ||
|
|
10407
|
+
!this.isReceiveOwnershipSnapshotStable(ownershipRevision)
|
|
10408
|
+
) {
|
|
10409
|
+
return;
|
|
10410
|
+
}
|
|
10411
|
+
const assignmentLeaders = new Map(leaders);
|
|
10412
|
+
const deliveryTargets = new Set(leaders.keys());
|
|
10413
|
+
if (source.extrasOwnershipRevision === ownershipRevision) {
|
|
10414
|
+
for (const [peer, sample] of source.assignmentExtraLeaders) {
|
|
10415
|
+
assignmentLeaders.set(peer, sample);
|
|
10416
|
+
}
|
|
10417
|
+
for (const peer of source.deliveryExtraTargets) {
|
|
10418
|
+
deliveryTargets.add(peer);
|
|
10419
|
+
}
|
|
10420
|
+
}
|
|
10421
|
+
const repairEntry = this.createEntryReplicatedForRepair({
|
|
10422
|
+
entry: source.entry,
|
|
10423
|
+
coordinates: source.coordinates,
|
|
10424
|
+
leaders: assignmentLeaders,
|
|
10425
|
+
replicas,
|
|
10426
|
+
});
|
|
10427
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
10428
|
+
for (const peer of assignmentLeaders.keys()) {
|
|
10429
|
+
deliveryTargets.add(peer);
|
|
10430
|
+
}
|
|
10431
|
+
for (const peer of deliveryTargets) {
|
|
10432
|
+
if (peer === selfHash) continue;
|
|
10433
|
+
if (!this.isReceiveOwnershipSnapshotStable(ownershipRevision)) return;
|
|
10434
|
+
try {
|
|
10435
|
+
this.queueAppendBackfill(
|
|
10436
|
+
peer,
|
|
10437
|
+
repairEntry,
|
|
10438
|
+
repairLifecycleController,
|
|
10439
|
+
);
|
|
10440
|
+
} catch (error) {
|
|
10441
|
+
if (this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
10442
|
+
logger.error(error);
|
|
10443
|
+
}
|
|
10444
|
+
}
|
|
10445
|
+
}
|
|
10446
|
+
} catch (error) {
|
|
10447
|
+
if (this.isRepairLifecycleActive(repairLifecycleController)) {
|
|
10448
|
+
logger.error(error);
|
|
10449
|
+
}
|
|
10450
|
+
}
|
|
10451
|
+
}
|
|
10452
|
+
|
|
10252
10453
|
private dispatchMaybeMissingEntries(
|
|
10253
10454
|
target: string,
|
|
10254
10455
|
entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
|
|
@@ -11894,6 +12095,12 @@ export class SharedLog<
|
|
|
11894
12095
|
let persistedPlanningRecord:
|
|
11895
12096
|
| PersistedDeliveryPlanningRecord<T, R>
|
|
11896
12097
|
| undefined;
|
|
12098
|
+
let persistedBackfillSource:
|
|
12099
|
+
| PersistedAppendBackfillSource<T, R>
|
|
12100
|
+
| undefined;
|
|
12101
|
+
let persistedBackfillLeaders: LeaderMap | undefined;
|
|
12102
|
+
let persistedBackfillOwnershipRevision: number | undefined;
|
|
12103
|
+
let localAppendProcessed = false;
|
|
11897
12104
|
if (persistedDelivery) {
|
|
11898
12105
|
(appendOptions as TrustedLogAppendOptions<T>).__peerbitOnLocalCommit = (
|
|
11899
12106
|
hashes,
|
|
@@ -11947,8 +12154,17 @@ export class SharedLog<
|
|
|
11947
12154
|
await this.processLocalAppend(processingEntry, result.removed, options, {
|
|
11948
12155
|
minReplicasValue,
|
|
11949
12156
|
appendFacts: persistedAppendCommit,
|
|
12157
|
+
// Persisted settlement must confirm each exact receiver generation before
|
|
12158
|
+
// using its transfer as receipt evidence. Keep the optimistic append path
|
|
12159
|
+
// out of that ordering decision.
|
|
12160
|
+
captureDeferredBackfillSource: persistedDelivery
|
|
12161
|
+
? (source) => {
|
|
12162
|
+
persistedBackfillSource = source;
|
|
12163
|
+
}
|
|
12164
|
+
: undefined,
|
|
11950
12165
|
ownershipLifecycleController,
|
|
11951
12166
|
});
|
|
12167
|
+
localAppendProcessed = true;
|
|
11952
12168
|
throwIfDeliveryAborted();
|
|
11953
12169
|
if (persistedDelivery && persistedDeadline) {
|
|
11954
12170
|
await this.settlePersistedDelivery(
|
|
@@ -11957,6 +12173,12 @@ export class SharedLog<
|
|
|
11957
12173
|
persistedDelivery,
|
|
11958
12174
|
ownershipLifecycleController,
|
|
11959
12175
|
persistedDeadline,
|
|
12176
|
+
true,
|
|
12177
|
+
(leadersByEntry, ownershipRevision) => {
|
|
12178
|
+
const leaders = leadersByEntry[0];
|
|
12179
|
+
persistedBackfillLeaders = leaders ? new Map(leaders) : undefined;
|
|
12180
|
+
persistedBackfillOwnershipRevision = ownershipRevision;
|
|
12181
|
+
},
|
|
11960
12182
|
);
|
|
11961
12183
|
}
|
|
11962
12184
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
@@ -11969,6 +12191,27 @@ export class SharedLog<
|
|
|
11969
12191
|
}
|
|
11970
12192
|
throw error;
|
|
11971
12193
|
} finally {
|
|
12194
|
+
if (
|
|
12195
|
+
persistedBackfillSource &&
|
|
12196
|
+
persistedBackfillLeaders &&
|
|
12197
|
+
persistedBackfillOwnershipRevision !== undefined &&
|
|
12198
|
+
localAppendProcessed
|
|
12199
|
+
) {
|
|
12200
|
+
// A persisted quorum changes the return condition, not the configured
|
|
12201
|
+
// replication degree. Reuse settlement's latest fresh leader plan rather
|
|
12202
|
+
// than carrying an optimistic target or extending the receipt deadline
|
|
12203
|
+
// with another plan. Fence best-effort repair to this append's ownership
|
|
12204
|
+
// generation, and never let it mask the primary result.
|
|
12205
|
+
try {
|
|
12206
|
+
this.queuePersistedAppendBackfill(
|
|
12207
|
+
persistedBackfillSource,
|
|
12208
|
+
persistedBackfillLeaders,
|
|
12209
|
+
minReplicasValue,
|
|
12210
|
+
ownershipLifecycleController,
|
|
12211
|
+
persistedBackfillOwnershipRevision,
|
|
12212
|
+
);
|
|
12213
|
+
} catch {}
|
|
12214
|
+
}
|
|
11972
12215
|
persistedDeadline?.dispose();
|
|
11973
12216
|
}
|
|
11974
12217
|
}
|
|
@@ -16080,6 +16323,9 @@ export class SharedLog<
|
|
|
16080
16323
|
minReplicasValue: number;
|
|
16081
16324
|
appendFacts?: PreparedAppendFacts;
|
|
16082
16325
|
deferHeadCoordinatePersistence?: boolean;
|
|
16326
|
+
captureDeferredBackfillSource?: (
|
|
16327
|
+
source: PersistedAppendBackfillSource<T, R>,
|
|
16328
|
+
) => void;
|
|
16083
16329
|
nativeAppendPlan?: NativeAppendEntryPlan<R>;
|
|
16084
16330
|
extraCoordinateDeleteHashes?: string[];
|
|
16085
16331
|
ownershipLifecycleController?: AbortController;
|
|
@@ -16223,7 +16469,41 @@ export class SharedLog<
|
|
|
16223
16469
|
ownershipLifecycleController,
|
|
16224
16470
|
);
|
|
16225
16471
|
|
|
16226
|
-
if (
|
|
16472
|
+
if (properties.captureDeferredBackfillSource) {
|
|
16473
|
+
let extras: Omit<
|
|
16474
|
+
PersistedAppendBackfillSource<T, R>,
|
|
16475
|
+
"entry" | "coordinates"
|
|
16476
|
+
> = {
|
|
16477
|
+
assignmentExtraLeaders: new Map(),
|
|
16478
|
+
deliveryExtraTargets: new Set(),
|
|
16479
|
+
};
|
|
16480
|
+
try {
|
|
16481
|
+
extras = await this.collectDeferredAppendBackfillExtras(
|
|
16482
|
+
entry,
|
|
16483
|
+
properties.minReplicasValue,
|
|
16484
|
+
leaders!,
|
|
16485
|
+
nativeDeliveryPlan,
|
|
16486
|
+
ownershipLifecycleController,
|
|
16487
|
+
);
|
|
16488
|
+
} catch (error) {
|
|
16489
|
+
if (this.isRepairLifecycleActive(ownershipLifecycleController)) {
|
|
16490
|
+
logger.error(error);
|
|
16491
|
+
}
|
|
16492
|
+
}
|
|
16493
|
+
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
16494
|
+
ownershipLifecycleController,
|
|
16495
|
+
);
|
|
16496
|
+
properties.captureDeferredBackfillSource({
|
|
16497
|
+
entry,
|
|
16498
|
+
coordinates: [...coordinates],
|
|
16499
|
+
...extras,
|
|
16500
|
+
});
|
|
16501
|
+
}
|
|
16502
|
+
|
|
16503
|
+
if (
|
|
16504
|
+
options?.target !== "none" &&
|
|
16505
|
+
!properties.captureDeferredBackfillSource
|
|
16506
|
+
) {
|
|
16227
16507
|
const hasDelivery = !(deliveryArg === undefined || deliveryArg === false);
|
|
16228
16508
|
|
|
16229
16509
|
if (target === "all" && hasDelivery) {
|
|
@@ -22498,6 +22778,12 @@ export class SharedLog<
|
|
|
22498
22778
|
this._peerSyncCapabilitySessions.get(lane.fromHash) ===
|
|
22499
22779
|
context.message.header.session &&
|
|
22500
22780
|
this._peerSyncCapabilityTimestamps.has(lane.fromHash) &&
|
|
22781
|
+
this._v2Receive.isCurrentActive({
|
|
22782
|
+
peerHash: lane.fromHash,
|
|
22783
|
+
peerSession: session,
|
|
22784
|
+
receiveEpoch: lane.receiveEpoch,
|
|
22785
|
+
senderTransportSession: context.message.header.session,
|
|
22786
|
+
}) &&
|
|
22501
22787
|
this.isRepairLifecycleActive(lane.ownershipLifecycleController)
|
|
22502
22788
|
);
|
|
22503
22789
|
}
|
|
@@ -23604,7 +23890,7 @@ export class SharedLog<
|
|
|
23604
23890
|
const peerHash = key.hashcode();
|
|
23605
23891
|
const peerSession = this._peerSessions.current(peerHash);
|
|
23606
23892
|
if (peerSession?.phase === "open") {
|
|
23607
|
-
this._v2Receive.
|
|
23893
|
+
this._v2Receive.ensureRequestProgress({
|
|
23608
23894
|
peerHash,
|
|
23609
23895
|
peerSession,
|
|
23610
23896
|
receiveEpoch: this._peerSessions.receiveEpoch(peerHash),
|
|
@@ -24866,6 +25152,7 @@ export class SharedLog<
|
|
|
24866
25152
|
options?: {
|
|
24867
25153
|
roleAge?: number;
|
|
24868
25154
|
candidates?: Iterable<string>;
|
|
25155
|
+
freshLeaderPlan?: boolean;
|
|
24869
25156
|
},
|
|
24870
25157
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
24871
25158
|
): Promise<LeaderSelectionContext> {
|
|
@@ -25027,6 +25314,7 @@ export class SharedLog<
|
|
|
25027
25314
|
options?: {
|
|
25028
25315
|
roleAge?: number;
|
|
25029
25316
|
candidates?: Iterable<string>;
|
|
25317
|
+
freshLeaderPlan?: boolean;
|
|
25030
25318
|
},
|
|
25031
25319
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
25032
25320
|
): Promise<Map<string, { intersecting: boolean }>> {
|
|
@@ -26447,8 +26735,14 @@ export class SharedLog<
|
|
|
26447
26735
|
receiveEpoch,
|
|
26448
26736
|
})
|
|
26449
26737
|
) {
|
|
26450
|
-
// Active
|
|
26451
|
-
//
|
|
26738
|
+
// Active and timed/in-flight cycles are no-ops. An unparked,
|
|
26739
|
+
// non-active generation with no worker can occur when its timer fires
|
|
26740
|
+
// behind a temporary receive gate; repair that exact session here.
|
|
26741
|
+
this._v2Receive.ensureRequestProgress({
|
|
26742
|
+
peerHash,
|
|
26743
|
+
peerSession,
|
|
26744
|
+
receiveEpoch,
|
|
26745
|
+
});
|
|
26452
26746
|
state.parkedSinceMs = undefined;
|
|
26453
26747
|
arm(intervalMs);
|
|
26454
26748
|
return;
|
|
@@ -947,6 +947,20 @@ export class ReplicationInfoV2ReceiveCoordinator {
|
|
|
947
947
|
);
|
|
948
948
|
}
|
|
949
949
|
|
|
950
|
+
/**
|
|
951
|
+
* Ensure an exact current request generation has a worker. In addition to
|
|
952
|
+
* resuming a bounded cycle that parked normally, this repairs a request whose
|
|
953
|
+
* timer fired while the host's receive gate was temporarily closed: that
|
|
954
|
+
* generation is neither parked nor active, but has no timer or send in flight.
|
|
955
|
+
*/
|
|
956
|
+
ensureRequestProgress(properties: {
|
|
957
|
+
peerHash: string;
|
|
958
|
+
peerSession: object;
|
|
959
|
+
receiveEpoch: object | null;
|
|
960
|
+
}): boolean {
|
|
961
|
+
return this.nudgeRequest(properties, false);
|
|
962
|
+
}
|
|
963
|
+
|
|
950
964
|
/**
|
|
951
965
|
* Resume only a request generation that exhausted its bounded retries.
|
|
952
966
|
* Wait/liveness callers may nudge recovery without invalidating an active,
|
|
@@ -957,18 +971,25 @@ export class ReplicationInfoV2ReceiveCoordinator {
|
|
|
957
971
|
peerSession: object;
|
|
958
972
|
receiveEpoch: object | null;
|
|
959
973
|
}): boolean {
|
|
974
|
+
return this.nudgeRequest(properties, true);
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
private nudgeRequest(
|
|
978
|
+
properties: {
|
|
979
|
+
peerHash: string;
|
|
980
|
+
peerSession: object;
|
|
981
|
+
receiveEpoch: object | null;
|
|
982
|
+
},
|
|
983
|
+
parkedOnly: boolean,
|
|
984
|
+
): boolean {
|
|
960
985
|
const state = this._receiveStates.get(properties.peerHash);
|
|
961
986
|
if (
|
|
962
987
|
!state ||
|
|
963
988
|
state.peerSession !== properties.peerSession ||
|
|
964
989
|
state.receiveEpoch !== properties.receiveEpoch ||
|
|
965
|
-
!this.
|
|
966
|
-
properties.peerHash,
|
|
967
|
-
properties.peerSession,
|
|
968
|
-
properties.receiveEpoch,
|
|
969
|
-
) ||
|
|
990
|
+
!this.isStateCurrent(state) ||
|
|
970
991
|
state.phase === "active" ||
|
|
971
|
-
!state.requestParked ||
|
|
992
|
+
(parkedOnly && !state.requestParked) ||
|
|
972
993
|
state.requestTimer !== undefined ||
|
|
973
994
|
state.requestInFlight !== undefined ||
|
|
974
995
|
this._reservedAdmissionsByPeer.has(properties.peerHash) ||
|
|
@@ -977,9 +998,13 @@ export class ReplicationInfoV2ReceiveCoordinator {
|
|
|
977
998
|
) {
|
|
978
999
|
return false;
|
|
979
1000
|
}
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
if (
|
|
1001
|
+
const restartBoundedCycle =
|
|
1002
|
+
state.requestParked || state.requestAttempts >= this.requestMaxAttempts;
|
|
1003
|
+
if (restartBoundedCycle) {
|
|
1004
|
+
state.requestAttempts = 0;
|
|
1005
|
+
state.requestsSinceCapabilityRefresh = 0;
|
|
1006
|
+
}
|
|
1007
|
+
if (restartBoundedCycle && !state.capabilityRefreshRequired) {
|
|
983
1008
|
// Only rotate the grant when it is genuinely stale: the peer's
|
|
984
1009
|
// capability rotation paths already flagged a refresh, and a missing
|
|
985
1010
|
// or transport-outdated local grant cannot authorize a request. A
|
|
@@ -1210,6 +1235,31 @@ export class ReplicationInfoV2ReceiveCoordinator {
|
|
|
1210
1235
|
);
|
|
1211
1236
|
}
|
|
1212
1237
|
|
|
1238
|
+
/**
|
|
1239
|
+
* Exact active receive generation used by durability-sensitive callers.
|
|
1240
|
+
* An open PeerSession alone is not evidence that its authoritative Full has
|
|
1241
|
+
* crossed the host apply lane, especially during a same-key reconnect.
|
|
1242
|
+
*/
|
|
1243
|
+
isCurrentActive(properties: {
|
|
1244
|
+
peerHash: string;
|
|
1245
|
+
peerSession: object;
|
|
1246
|
+
receiveEpoch: object | null;
|
|
1247
|
+
senderTransportSession: bigint;
|
|
1248
|
+
}): boolean {
|
|
1249
|
+
const state = this._receiveStates.get(properties.peerHash);
|
|
1250
|
+
return (
|
|
1251
|
+
state !== undefined &&
|
|
1252
|
+
state.peerHash === properties.peerHash &&
|
|
1253
|
+
state.peerSession === properties.peerSession &&
|
|
1254
|
+
state.receiveEpoch === properties.receiveEpoch &&
|
|
1255
|
+
state.senderTransportSession === properties.senderTransportSession &&
|
|
1256
|
+
state.phase === "active" &&
|
|
1257
|
+
state.reservedAdmission === undefined &&
|
|
1258
|
+
!this._reservedAdmissionsByPeer.has(properties.peerHash) &&
|
|
1259
|
+
this.isStateCurrent(state)
|
|
1260
|
+
);
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1213
1263
|
commit(admission: ReplicationInfoV2ReceiveAdmission): boolean {
|
|
1214
1264
|
if (admission.committed) {
|
|
1215
1265
|
return this.isAdmissionCurrent(admission);
|
|
@@ -1269,9 +1319,12 @@ export class ReplicationInfoV2ReceiveCoordinator {
|
|
|
1269
1319
|
const state = this._receiveStates.get(properties.from.hashcode());
|
|
1270
1320
|
if (
|
|
1271
1321
|
!state ||
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1322
|
+
!this.isCurrentActive({
|
|
1323
|
+
peerHash: properties.from.hashcode(),
|
|
1324
|
+
peerSession: properties.peerSession,
|
|
1325
|
+
receiveEpoch: properties.receiveEpoch,
|
|
1326
|
+
senderTransportSession: properties.senderTransportSession,
|
|
1327
|
+
}) ||
|
|
1275
1328
|
!state.target.equals(properties.from) ||
|
|
1276
1329
|
!state.receiverBinding ||
|
|
1277
1330
|
!bytesEqual(request.receiverChallenge, state.receiverBinding) ||
|
|
@@ -1279,8 +1332,7 @@ export class ReplicationInfoV2ReceiveCoordinator {
|
|
|
1279
1332
|
!bytesEqual(request.senderEpoch, state.senderEpoch) ||
|
|
1280
1333
|
request.sequence <= 0n ||
|
|
1281
1334
|
state.lastSequence === undefined ||
|
|
1282
|
-
state.lastSequence < request.sequence
|
|
1283
|
-
!this.isStateCurrent(state)
|
|
1335
|
+
state.lastSequence < request.sequence
|
|
1284
1336
|
) {
|
|
1285
1337
|
return undefined;
|
|
1286
1338
|
}
|