@peerbit/shared-log 13.2.32 → 13.2.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +6 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +613 -123
- package/dist/src/index.js.map +1 -1
- package/dist/src/replication-info-v2-binding.d.ts +16 -0
- package/dist/src/replication-info-v2-binding.d.ts.map +1 -0
- package/dist/src/replication-info-v2-binding.js +30 -0
- package/dist/src/replication-info-v2-binding.js.map +1 -0
- package/dist/src/replication-info-v2-receive.d.ts +250 -0
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -0
- package/dist/src/replication-info-v2-receive.js +1089 -0
- package/dist/src/replication-info-v2-receive.js.map +1 -0
- package/dist/src/replication-info-v2-send.d.ts +25 -17
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +170 -86
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/package.json +8 -8
- package/src/index.ts +897 -260
- package/src/replication-info-v2-binding.ts +44 -0
- package/src/replication-info-v2-receive.ts +1577 -0
- package/src/replication-info-v2-send.ts +223 -109
package/src/index.ts
CHANGED
|
@@ -130,6 +130,14 @@ import {
|
|
|
130
130
|
type CheckedPruneRetryIdentity,
|
|
131
131
|
type CheckedPruneRetryState,
|
|
132
132
|
} from "./checked-prune.js";
|
|
133
|
+
import {
|
|
134
|
+
CoordinatePersistenceCoordinator,
|
|
135
|
+
type MaybePromise,
|
|
136
|
+
combineCoordinateDeleteHashes,
|
|
137
|
+
isPromiseLike,
|
|
138
|
+
mapMaybePromise,
|
|
139
|
+
normalizedHashValues,
|
|
140
|
+
} from "./coordinate-persistence.js";
|
|
133
141
|
import { type CPUUsage, CPUUsageIntervalLag } from "./cpu.js";
|
|
134
142
|
import {
|
|
135
143
|
type DebouncedAccumulatorMap,
|
|
@@ -181,14 +189,6 @@ import {
|
|
|
181
189
|
materializeVerifiedRawExchangeHeadsMessage,
|
|
182
190
|
} from "./exchange-heads.js";
|
|
183
191
|
import { FanoutEnvelope } from "./fanout-envelope.js";
|
|
184
|
-
import {
|
|
185
|
-
CoordinatePersistenceCoordinator,
|
|
186
|
-
type MaybePromise,
|
|
187
|
-
combineCoordinateDeleteHashes,
|
|
188
|
-
isPromiseLike,
|
|
189
|
-
mapMaybePromise,
|
|
190
|
-
normalizedHashValues,
|
|
191
|
-
} from "./coordinate-persistence.js";
|
|
192
192
|
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
193
193
|
import {
|
|
194
194
|
MAX_U32,
|
|
@@ -201,10 +201,7 @@ import { JoinWarmupCoordinator, type WarmupSession } from "./join-warmup.js";
|
|
|
201
201
|
import { LeaderPlanCache } from "./leader-plan-cache.js";
|
|
202
202
|
import { TransportMessage } from "./message.js";
|
|
203
203
|
import { NativeBackboneWriteThroughBlockStore } from "./native-write-through-block-store.js";
|
|
204
|
-
import {
|
|
205
|
-
type PeerSession,
|
|
206
|
-
PeerSessionRegistry,
|
|
207
|
-
} from "./peer-session.js";
|
|
204
|
+
import { type PeerSession, PeerSessionRegistry } from "./peer-session.js";
|
|
208
205
|
import { PIDReplicationController } from "./pid.js";
|
|
209
206
|
import {
|
|
210
207
|
type EntryReplicated,
|
|
@@ -237,7 +234,6 @@ import {
|
|
|
237
234
|
ReplicationAnnouncementCoordinator,
|
|
238
235
|
isTransientReplicationAnnouncementError,
|
|
239
236
|
} from "./replication-announcement.js";
|
|
240
|
-
import { ReplicationInfoV2SendCoordinator } from "./replication-info-v2-send.js";
|
|
241
237
|
import {
|
|
242
238
|
type ReplicationDomainHash,
|
|
243
239
|
createReplicationDomainHash,
|
|
@@ -252,18 +248,24 @@ import {
|
|
|
252
248
|
type ReplicationDomain,
|
|
253
249
|
type ReplicationDomainConstructor,
|
|
254
250
|
} from "./replication-domain.js";
|
|
251
|
+
import { ReplicationInfoV2ReceiveCoordinator } from "./replication-info-v2-receive.js";
|
|
252
|
+
import { ReplicationInfoV2SendCoordinator } from "./replication-info-v2-send.js";
|
|
255
253
|
import {
|
|
256
254
|
AbsoluteReplicas,
|
|
255
|
+
AddedReplicationInfoV2Message,
|
|
257
256
|
AddedReplicationSegmentMessage,
|
|
258
257
|
AllReplicatingSegmentsMessage,
|
|
258
|
+
FullReplicationInfoV2Message,
|
|
259
259
|
MinReplicas,
|
|
260
260
|
ReplicationError,
|
|
261
|
+
type ReplicationInfoV2Message,
|
|
261
262
|
type ReplicationLimits,
|
|
262
263
|
ReplicationPingMessage,
|
|
263
264
|
RequestReplicationInfoMessage,
|
|
264
265
|
RequestReplicationInfoV2Message,
|
|
265
266
|
ResponseRoleMessage,
|
|
266
267
|
StoppedReplicating,
|
|
268
|
+
StoppedReplicationInfoV2Message,
|
|
267
269
|
decodeReplicas,
|
|
268
270
|
encodeReplicas,
|
|
269
271
|
isReplicationInfoV2Message,
|
|
@@ -354,6 +356,7 @@ type ReceiveLaneContext = {
|
|
|
354
356
|
fromHash: string;
|
|
355
357
|
session: PeerSession | null;
|
|
356
358
|
lifecycleController: AbortController | undefined;
|
|
359
|
+
ownershipLifecycleController: AbortController;
|
|
357
360
|
receiveEpoch: object | null;
|
|
358
361
|
syncProfile: SyncProfileFn | undefined;
|
|
359
362
|
lease: PeerReceiveLease;
|
|
@@ -560,6 +563,7 @@ type ReplicationRangeDeletionOutcome<R extends "u32" | "u64"> = {
|
|
|
560
563
|
removed: ReplicationRangeIndexable<R>[];
|
|
561
564
|
retained: ReplicationRangeIndexable<R>[];
|
|
562
565
|
ownerHasRanges: boolean;
|
|
566
|
+
rollback: () => Promise<void>;
|
|
563
567
|
error?: unknown;
|
|
564
568
|
};
|
|
565
569
|
|
|
@@ -632,7 +636,8 @@ export type NativeBackboneCoordinateRollback<R extends "u32" | "u64"> = {
|
|
|
632
636
|
generations: Map<string, number>;
|
|
633
637
|
};
|
|
634
638
|
|
|
635
|
-
export type RepairDispatchEntry<R extends "u32" | "u64"> =
|
|
639
|
+
export type RepairDispatchEntry<R extends "u32" | "u64"> =
|
|
640
|
+
ResidentCoordinateEntry<R>;
|
|
636
641
|
|
|
637
642
|
type PreparedLocalAppendCommit<R extends "u32" | "u64"> = {
|
|
638
643
|
hash: string;
|
|
@@ -1091,7 +1096,6 @@ const isReplicationOptionsDependentOnPreviousState = async (
|
|
|
1091
1096
|
return false;
|
|
1092
1097
|
};
|
|
1093
1098
|
|
|
1094
|
-
|
|
1095
1099
|
export interface IndexableDomain<R extends "u32" | "u64"> {
|
|
1096
1100
|
numbers: Numbers<R>;
|
|
1097
1101
|
constructorEntry: new (properties: {
|
|
@@ -1990,7 +1994,8 @@ export class SharedLog<
|
|
|
1990
1994
|
}
|
|
1991
1995
|
|
|
1992
1996
|
get _nativeBackboneCoordinateJournalLastFlushMs(): number {
|
|
1993
|
-
return this._coordinates
|
|
1997
|
+
return this._coordinates
|
|
1998
|
+
?._nativeBackboneCoordinateJournalLastFlushMs as number;
|
|
1994
1999
|
}
|
|
1995
2000
|
|
|
1996
2001
|
set _nativeBackboneCoordinateJournalLastFlushMs(value: number) {
|
|
@@ -3077,7 +3082,8 @@ export class SharedLog<
|
|
|
3077
3082
|
if (!lowerMarkerCommitted) {
|
|
3078
3083
|
if (intent.coordinates.length > 0) {
|
|
3079
3084
|
const mutationGenerations =
|
|
3080
|
-
(this._coordinates._nativeCoordinateMutationGenerations ??=
|
|
3085
|
+
(this._coordinates._nativeCoordinateMutationGenerations ??=
|
|
3086
|
+
new Map());
|
|
3081
3087
|
const rollback: NativeBackboneCoordinateRollback<R> = {
|
|
3082
3088
|
hashes: new Set(),
|
|
3083
3089
|
entries: new Map(),
|
|
@@ -3109,7 +3115,10 @@ export class SharedLog<
|
|
|
3109
3115
|
);
|
|
3110
3116
|
}
|
|
3111
3117
|
}
|
|
3112
|
-
await this._coordinates.rollbackNativeBackboneCoordinateAppendDurably(
|
|
3118
|
+
await this._coordinates.rollbackNativeBackboneCoordinateAppendDurably(
|
|
3119
|
+
"",
|
|
3120
|
+
rollback,
|
|
3121
|
+
);
|
|
3113
3122
|
}
|
|
3114
3123
|
for (const document of intent.documents) {
|
|
3115
3124
|
this.restoreNativeBackboneDocument({
|
|
@@ -3234,6 +3243,7 @@ export class SharedLog<
|
|
|
3234
3243
|
| ReturnType<typeof debounceFixedInterval>
|
|
3235
3244
|
| undefined;
|
|
3236
3245
|
private _announcements!: ReplicationAnnouncementCoordinator<R>;
|
|
3246
|
+
private _v2Receive!: ReplicationInfoV2ReceiveCoordinator;
|
|
3237
3247
|
private _v2Send!: ReplicationInfoV2SendCoordinator<R>;
|
|
3238
3248
|
|
|
3239
3249
|
// A fn for debouncing the calls for pruning
|
|
@@ -3368,16 +3378,20 @@ export class SharedLog<
|
|
|
3368
3378
|
queueCurrentReplicationStateAnnouncementRepair: () =>
|
|
3369
3379
|
this._announcements.queueCurrentReplicationStateAnnouncementRepair(),
|
|
3370
3380
|
queueCurrentReplicationStateAnnouncementRetry: (error: unknown) =>
|
|
3371
|
-
this._announcements.queueCurrentReplicationStateAnnouncementRetry(
|
|
3381
|
+
this._announcements.queueCurrentReplicationStateAnnouncementRetry(
|
|
3382
|
+
error,
|
|
3383
|
+
),
|
|
3372
3384
|
enqueueReplicationInfoV2: (message) => this._v2Send.enqueue(message),
|
|
3373
3385
|
isClosed: () => this.closed,
|
|
3374
3386
|
getCloseSignal: () => this._closeController.signal,
|
|
3375
3387
|
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
3376
3388
|
validatePersistedReplicationRangeSnapshot: (ranges) =>
|
|
3377
3389
|
this.validatePersistedReplicationRangeSnapshot(ranges),
|
|
3378
|
-
getSubscribers: () =>
|
|
3390
|
+
getSubscribers: () =>
|
|
3391
|
+
this.node.services.pubsub.getSubscribers(this.topic),
|
|
3379
3392
|
getSelfHash: () => this.node.identity.publicKey.hashcode(),
|
|
3380
|
-
isBlockedPeer: (hash) =>
|
|
3393
|
+
isBlockedPeer: (hash) =>
|
|
3394
|
+
this._peerSessions.isReplicationInfoBlocked(hash),
|
|
3381
3395
|
getRpc: () => this.rpc,
|
|
3382
3396
|
captureReplicationOwnershipLifecycle: () =>
|
|
3383
3397
|
this.captureReplicationOwnershipLifecycle(),
|
|
@@ -3401,6 +3415,8 @@ export class SharedLog<
|
|
|
3401
3415
|
validatePersistedReplicationRangeSnapshot: (ranges) =>
|
|
3402
3416
|
this.validatePersistedReplicationRangeSnapshot(ranges),
|
|
3403
3417
|
isClosed: () => this.closed,
|
|
3418
|
+
isPeerSessionCurrent: (peerHash, peerSession) =>
|
|
3419
|
+
this._peerSessions.isCurrent(peerHash, peerSession),
|
|
3404
3420
|
isPeerSessionOpen: (peerHash, peerSession) =>
|
|
3405
3421
|
this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
3406
3422
|
(peerSession as PeerSession).phase === "open" &&
|
|
@@ -3413,6 +3429,122 @@ export class SharedLog<
|
|
|
3413
3429
|
});
|
|
3414
3430
|
}
|
|
3415
3431
|
|
|
3432
|
+
private replicationInfoV2ReceiveCapabilities(): number {
|
|
3433
|
+
return (
|
|
3434
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
3435
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
|
|
3436
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
|
|
3437
|
+
(this._logProperties?.sync?.rawExchangeHeads === true
|
|
3438
|
+
? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
|
|
3439
|
+
: 0)
|
|
3440
|
+
);
|
|
3441
|
+
}
|
|
3442
|
+
|
|
3443
|
+
private async advertiseReplicationInfoV2ReceiveCapability(properties: {
|
|
3444
|
+
target: PublicSignKey;
|
|
3445
|
+
peerSession: PeerSession;
|
|
3446
|
+
receiveEpoch: object | null;
|
|
3447
|
+
signal: AbortSignal;
|
|
3448
|
+
}): Promise<
|
|
3449
|
+
{ receiverTransportSession: bigint; requestNotBeforeMs: number } | undefined
|
|
3450
|
+
> {
|
|
3451
|
+
const peerHash = properties.target.hashcode();
|
|
3452
|
+
const receiverTransportSession = BigInt(
|
|
3453
|
+
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3454
|
+
);
|
|
3455
|
+
await this.rpc.send(
|
|
3456
|
+
new SyncCapabilitiesMessage({
|
|
3457
|
+
capabilities: this.replicationInfoV2ReceiveCapabilities(),
|
|
3458
|
+
}),
|
|
3459
|
+
{
|
|
3460
|
+
mode: new AcknowledgeDelivery({
|
|
3461
|
+
redundancy: 1,
|
|
3462
|
+
to: [properties.target],
|
|
3463
|
+
}),
|
|
3464
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
3465
|
+
signal: properties.signal,
|
|
3466
|
+
},
|
|
3467
|
+
);
|
|
3468
|
+
if (
|
|
3469
|
+
properties.signal.aborted ||
|
|
3470
|
+
this.closed ||
|
|
3471
|
+
!this._peerSessions.isCurrent(peerHash, properties.peerSession) ||
|
|
3472
|
+
properties.peerSession.phase !== "open" ||
|
|
3473
|
+
!properties.peerSession.isActive() ||
|
|
3474
|
+
!this._peerSessions.isReceiveEpochCurrent(
|
|
3475
|
+
peerHash,
|
|
3476
|
+
properties.receiveEpoch,
|
|
3477
|
+
) ||
|
|
3478
|
+
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
3479
|
+
!this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
|
|
3480
|
+
BigInt(
|
|
3481
|
+
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3482
|
+
) !== receiverTransportSession
|
|
3483
|
+
) {
|
|
3484
|
+
return undefined;
|
|
3485
|
+
}
|
|
3486
|
+
return {
|
|
3487
|
+
receiverTransportSession,
|
|
3488
|
+
requestNotBeforeMs: Date.now(),
|
|
3489
|
+
};
|
|
3490
|
+
}
|
|
3491
|
+
|
|
3492
|
+
private createReplicationInfoV2ReceiveCoordinator(): ReplicationInfoV2ReceiveCoordinator {
|
|
3493
|
+
return new ReplicationInfoV2ReceiveCoordinator({
|
|
3494
|
+
getSelfKey: () => this.node.identity.publicKey,
|
|
3495
|
+
getReceiverTransportSession: () =>
|
|
3496
|
+
BigInt(
|
|
3497
|
+
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3498
|
+
),
|
|
3499
|
+
isClosed: () => this.closed,
|
|
3500
|
+
isPeerSessionCurrent: (peerHash, peerSession) =>
|
|
3501
|
+
this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
3502
|
+
(peerSession as PeerSession).phase === "open" &&
|
|
3503
|
+
(peerSession as PeerSession).isActive(),
|
|
3504
|
+
isReceiveEpochCurrent: (peerHash, receiveEpoch) =>
|
|
3505
|
+
this._peerSessions.isReceiveEpochCurrent(peerHash, receiveEpoch),
|
|
3506
|
+
isPeerStateCurrent: (peerHash, peerSession, receiveEpoch) =>
|
|
3507
|
+
this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
3508
|
+
(peerSession as PeerSession).phase === "open" &&
|
|
3509
|
+
(peerSession as PeerSession).isActive() &&
|
|
3510
|
+
this._peerSessions.isReceiveEpochCurrent(peerHash, receiveEpoch) &&
|
|
3511
|
+
!this._peerSessions.isReplicationInfoBlocked(peerHash) &&
|
|
3512
|
+
this._peerSessions.isReceiveCleanupGateOpen(peerHash),
|
|
3513
|
+
isSenderTransportSessionCurrent: (peerHash, senderTransportSession) =>
|
|
3514
|
+
this._peerSyncCapabilitySessions.get(peerHash) ===
|
|
3515
|
+
senderTransportSession,
|
|
3516
|
+
sendRequest: async (request, target, signal) => {
|
|
3517
|
+
await this.rpc.send(request, {
|
|
3518
|
+
mode: new AcknowledgeDelivery({ redundancy: 1, to: [target] }),
|
|
3519
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
3520
|
+
signal,
|
|
3521
|
+
});
|
|
3522
|
+
},
|
|
3523
|
+
refreshLocalCapability: (properties) =>
|
|
3524
|
+
this.advertiseReplicationInfoV2ReceiveCapability({
|
|
3525
|
+
target: properties.target,
|
|
3526
|
+
peerSession: properties.peerSession as PeerSession,
|
|
3527
|
+
receiveEpoch: properties.receiveEpoch,
|
|
3528
|
+
signal: properties.signal,
|
|
3529
|
+
}),
|
|
3530
|
+
onRequestError: (error) => {
|
|
3531
|
+
if (
|
|
3532
|
+
isNotStartedError(error as Error) ||
|
|
3533
|
+
(this.closed && error instanceof AbortError)
|
|
3534
|
+
) {
|
|
3535
|
+
return;
|
|
3536
|
+
}
|
|
3537
|
+
logger.trace(
|
|
3538
|
+
`Replication-info V2 recovery request failed: ${
|
|
3539
|
+
(error as Error)?.message ?? String(error)
|
|
3540
|
+
}`,
|
|
3541
|
+
);
|
|
3542
|
+
},
|
|
3543
|
+
onLocalCapabilityError: (error) =>
|
|
3544
|
+
this.handleReplicationLifecycleSendError(error),
|
|
3545
|
+
});
|
|
3546
|
+
}
|
|
3547
|
+
|
|
3416
3548
|
private createReplicatorLivenessMonitor(): ReplicatorLivenessMonitor {
|
|
3417
3549
|
return new ReplicatorLivenessMonitor({
|
|
3418
3550
|
// Sweep-driven probes and activity marks dispatch through the owner so
|
|
@@ -3443,7 +3575,8 @@ export class SharedLog<
|
|
|
3443
3575
|
}),
|
|
3444
3576
|
);
|
|
3445
3577
|
},
|
|
3446
|
-
isBlockedPeer: (hash) =>
|
|
3578
|
+
isBlockedPeer: (hash) =>
|
|
3579
|
+
this._peerSessions.isReplicationInfoBlocked(hash),
|
|
3447
3580
|
scheduleReplicationInfoRequests: (peer, replicationLifecycleController) =>
|
|
3448
3581
|
this.scheduleReplicationInfoRequests(
|
|
3449
3582
|
peer,
|
|
@@ -3537,6 +3670,7 @@ export class SharedLog<
|
|
|
3537
3670
|
this._replicatorJoinEmitted = new Set();
|
|
3538
3671
|
this._replicatorsReconciled = false;
|
|
3539
3672
|
this._liveness = this.createReplicatorLivenessMonitor();
|
|
3673
|
+
this._v2Receive = this.createReplicationInfoV2ReceiveCoordinator();
|
|
3540
3674
|
this._v2Send = this.createReplicationInfoV2SendCoordinator();
|
|
3541
3675
|
this._announcements = this.createReplicationAnnouncementCoordinator();
|
|
3542
3676
|
this.pendingMaturity = new Map();
|
|
@@ -4049,13 +4183,26 @@ export class SharedLog<
|
|
|
4049
4183
|
} = properties;
|
|
4050
4184
|
if (openingSession) {
|
|
4051
4185
|
const previous = this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
4186
|
+
if (
|
|
4187
|
+
transportSession !== undefined &&
|
|
4188
|
+
timestamp !== undefined &&
|
|
4189
|
+
previous?.epoch === openingSession &&
|
|
4190
|
+
previous.timestamp !== undefined &&
|
|
4191
|
+
previous.transportSession !== transportSession &&
|
|
4192
|
+
timestamp <= previous.timestamp
|
|
4193
|
+
) {
|
|
4194
|
+
return false;
|
|
4195
|
+
}
|
|
4052
4196
|
if (
|
|
4053
4197
|
transportSession !== undefined &&
|
|
4054
4198
|
timestamp !== undefined &&
|
|
4055
4199
|
previous?.epoch === openingSession &&
|
|
4056
4200
|
previous.transportSession === transportSession
|
|
4057
4201
|
) {
|
|
4058
|
-
if (
|
|
4202
|
+
if (
|
|
4203
|
+
previous.timestamp !== undefined &&
|
|
4204
|
+
timestamp < previous.timestamp
|
|
4205
|
+
) {
|
|
4059
4206
|
return false;
|
|
4060
4207
|
}
|
|
4061
4208
|
this._openingSyncCapabilitiesByPeer.set(peerHash, {
|
|
@@ -4085,6 +4232,7 @@ export class SharedLog<
|
|
|
4085
4232
|
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
4086
4233
|
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
4087
4234
|
this._v2Send.advancePeerCapability(peerHash);
|
|
4235
|
+
this._v2Receive.revokePeerCapability(peerHash);
|
|
4088
4236
|
return true;
|
|
4089
4237
|
}
|
|
4090
4238
|
|
|
@@ -4092,22 +4240,23 @@ export class SharedLog<
|
|
|
4092
4240
|
const previousTimestamp = this._peerSyncCapabilityTimestamps.get(peerHash);
|
|
4093
4241
|
const sameTransportSession = previousSession === transportSession;
|
|
4094
4242
|
if (
|
|
4095
|
-
sameTransportSession &&
|
|
4096
4243
|
previousTimestamp !== undefined &&
|
|
4097
|
-
timestamp < previousTimestamp
|
|
4244
|
+
((sameTransportSession && timestamp < previousTimestamp) ||
|
|
4245
|
+
(!sameTransportSession && timestamp <= previousTimestamp))
|
|
4098
4246
|
) {
|
|
4099
4247
|
return false;
|
|
4100
4248
|
}
|
|
4101
|
-
const previousCapabilities =
|
|
4102
|
-
this._peerSyncCapabilities.get(peerHash) ?? 0;
|
|
4249
|
+
const previousCapabilities = this._peerSyncCapabilities.get(peerHash) ?? 0;
|
|
4103
4250
|
const nextCapabilities = sameTransportSession
|
|
4104
4251
|
? previousCapabilities | capabilities
|
|
4105
4252
|
: capabilities;
|
|
4253
|
+
const senderGrantCapabilityMask =
|
|
4254
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
4255
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY;
|
|
4106
4256
|
const generationAdvanced =
|
|
4107
4257
|
!sameTransportSession ||
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
nextCapabilities !== previousCapabilities;
|
|
4258
|
+
(previousCapabilities & senderGrantCapabilityMask) !==
|
|
4259
|
+
(nextCapabilities & senderGrantCapabilityMask);
|
|
4111
4260
|
this._peerSyncCapabilities.set(peerHash, nextCapabilities);
|
|
4112
4261
|
this._peerSyncCapabilitySessions.set(peerHash, transportSession);
|
|
4113
4262
|
this._peerSyncCapabilityTimestamps.set(
|
|
@@ -4124,6 +4273,32 @@ export class SharedLog<
|
|
|
4124
4273
|
return true;
|
|
4125
4274
|
}
|
|
4126
4275
|
|
|
4276
|
+
private promoteReplicationInfoV2ReceiveCapability(
|
|
4277
|
+
target: PublicSignKey,
|
|
4278
|
+
peerSession: PeerSession,
|
|
4279
|
+
): boolean {
|
|
4280
|
+
const peerHash = target.hashcode();
|
|
4281
|
+
const senderTransportSession =
|
|
4282
|
+
this._peerSyncCapabilitySessions.get(peerHash);
|
|
4283
|
+
const capabilityTimestamp =
|
|
4284
|
+
this._peerSyncCapabilityTimestamps.get(peerHash);
|
|
4285
|
+
if (
|
|
4286
|
+
senderTransportSession === undefined ||
|
|
4287
|
+
capabilityTimestamp === undefined
|
|
4288
|
+
) {
|
|
4289
|
+
return false;
|
|
4290
|
+
}
|
|
4291
|
+
return this._v2Receive.observeCapability({
|
|
4292
|
+
peerHash,
|
|
4293
|
+
target,
|
|
4294
|
+
peerSession,
|
|
4295
|
+
receiveEpoch: this._peerSessions.receiveEpoch(peerHash),
|
|
4296
|
+
capabilities: this._peerSyncCapabilities.get(peerHash) ?? 0,
|
|
4297
|
+
senderTransportSession,
|
|
4298
|
+
capabilityTimestamp,
|
|
4299
|
+
});
|
|
4300
|
+
}
|
|
4301
|
+
|
|
4127
4302
|
/**
|
|
4128
4303
|
* Live append gossip may use the raw exchange-heads path only when we
|
|
4129
4304
|
* opted into raw sync and every remote recipient advertised raw capability
|
|
@@ -5001,7 +5176,8 @@ export class SharedLog<
|
|
|
5001
5176
|
|
|
5002
5177
|
private isReceiveOwnershipSnapshotStable(revision: number): boolean {
|
|
5003
5178
|
return (
|
|
5004
|
-
(this._instanceLifecycle?._receiveOwnershipMutationAdmissions ?? 0) ===
|
|
5179
|
+
(this._instanceLifecycle?._receiveOwnershipMutationAdmissions ?? 0) ===
|
|
5180
|
+
0 &&
|
|
5005
5181
|
(this._instanceLifecycle?._receiveOwnershipRevision ?? 0) === revision
|
|
5006
5182
|
);
|
|
5007
5183
|
}
|
|
@@ -5249,7 +5425,8 @@ export class SharedLog<
|
|
|
5249
5425
|
pending: PendingIHave<T>,
|
|
5250
5426
|
entry: Entry<T>,
|
|
5251
5427
|
): void {
|
|
5252
|
-
const replicationLifecycleController =
|
|
5428
|
+
const replicationLifecycleController =
|
|
5429
|
+
this._instanceLifecycle?.membershipLifecycleController;
|
|
5253
5430
|
if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
5254
5431
|
if (this._pendingIHave.get(entry.hash) === pending) {
|
|
5255
5432
|
pending.clear();
|
|
@@ -6157,6 +6334,7 @@ export class SharedLog<
|
|
|
6157
6334
|
};
|
|
6158
6335
|
let removed = false;
|
|
6159
6336
|
let removalCallCompleted = false;
|
|
6337
|
+
let replicationInfoRecoveryEpochAdvanced = false;
|
|
6160
6338
|
|
|
6161
6339
|
// Replication-info updates already serialize per peer. Put the hash-wide
|
|
6162
6340
|
// removal on the same queue so a newer reset cannot be deleted underneath
|
|
@@ -6246,6 +6424,13 @@ export class SharedLog<
|
|
|
6246
6424
|
checkedPruneCoordinator,
|
|
6247
6425
|
);
|
|
6248
6426
|
}
|
|
6427
|
+
if (!isMe) {
|
|
6428
|
+
// This is the last synchronous current-generation boundary
|
|
6429
|
+
// before destructive work. Fence parked receives now because
|
|
6430
|
+
// the coherent delete can durably mutate and then fail or poison.
|
|
6431
|
+
this.advanceReplicationInfoRecoveryEpoch(keyHash);
|
|
6432
|
+
replicationInfoRecoveryEpochAdvanced = true;
|
|
6433
|
+
}
|
|
6249
6434
|
const deletion = await this.deleteReplicationRangesCoherently(
|
|
6250
6435
|
deleted,
|
|
6251
6436
|
keyHash,
|
|
@@ -6269,7 +6454,6 @@ export class SharedLog<
|
|
|
6269
6454
|
}
|
|
6270
6455
|
return;
|
|
6271
6456
|
}
|
|
6272
|
-
|
|
6273
6457
|
if (options?.noEvent !== true && deleted.length > 0) {
|
|
6274
6458
|
const publicKey = toLocalPublicSignKey(key);
|
|
6275
6459
|
if (publicKey) {
|
|
@@ -6305,11 +6489,6 @@ export class SharedLog<
|
|
|
6305
6489
|
await cleanupDisconnectedPeer();
|
|
6306
6490
|
|
|
6307
6491
|
if (!isMe) {
|
|
6308
|
-
// Replication-info handlers release their receive lease before joining
|
|
6309
|
-
// this lane. Fence every handler queued behind this successful removal,
|
|
6310
|
-
// regardless of whether it came from liveness, startup pruning, or an
|
|
6311
|
-
// unsubscribe transition.
|
|
6312
|
-
this.advanceReplicationInfoRecoveryEpoch(keyHash);
|
|
6313
6492
|
this.rebalanceParticipationDebounced?.call();
|
|
6314
6493
|
}
|
|
6315
6494
|
removed = true;
|
|
@@ -6347,6 +6526,30 @@ export class SharedLog<
|
|
|
6347
6526
|
removalCallCompleted = true;
|
|
6348
6527
|
} finally {
|
|
6349
6528
|
releaseReceiveCleanupGate?.();
|
|
6529
|
+
if (
|
|
6530
|
+
replicationInfoRecoveryEpochAdvanced &&
|
|
6531
|
+
ownsReplicationOwnershipLifecycle() &&
|
|
6532
|
+
ownsReplicationLifecycle() &&
|
|
6533
|
+
ownsSubscriptionEpoch()
|
|
6534
|
+
) {
|
|
6535
|
+
// The first recovery arm ran while receive admission was fenced.
|
|
6536
|
+
// Re-arm after releasing the cleanup gate so a still-open peer can
|
|
6537
|
+
// deliver the authoritative Full that completes recovery.
|
|
6538
|
+
const peerSession = this._peerSessions.current(keyHash);
|
|
6539
|
+
if (peerSession?.phase === "open") {
|
|
6540
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(keyHash);
|
|
6541
|
+
this._v2Receive.advanceRecovery({
|
|
6542
|
+
peerHash: keyHash,
|
|
6543
|
+
peerSession,
|
|
6544
|
+
receiveEpoch,
|
|
6545
|
+
});
|
|
6546
|
+
this._v2Receive.reAdvertiseLocalCapabilityForRecovery({
|
|
6547
|
+
peerHash: keyHash,
|
|
6548
|
+
peerSession,
|
|
6549
|
+
receiveEpoch,
|
|
6550
|
+
});
|
|
6551
|
+
}
|
|
6552
|
+
}
|
|
6350
6553
|
if (isSpeculativePeerRemoval) {
|
|
6351
6554
|
const cancelledByFreshActivity =
|
|
6352
6555
|
removalCallCompleted &&
|
|
@@ -6422,6 +6625,7 @@ export class SharedLog<
|
|
|
6422
6625
|
options?: {
|
|
6423
6626
|
onRemoved?: (ranges: ReplicationRangeIndexable<R>[]) => void;
|
|
6424
6627
|
shouldRemove?: () => boolean;
|
|
6628
|
+
onDurableRemoveCommitted?: () => boolean | void;
|
|
6425
6629
|
},
|
|
6426
6630
|
replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
6427
6631
|
): Promise<boolean> {
|
|
@@ -6440,6 +6644,7 @@ export class SharedLog<
|
|
|
6440
6644
|
options?: {
|
|
6441
6645
|
onRemoved?: (ranges: ReplicationRangeIndexable<R>[]) => void;
|
|
6442
6646
|
shouldRemove?: () => boolean;
|
|
6647
|
+
onDurableRemoveCommitted?: () => boolean | void;
|
|
6443
6648
|
},
|
|
6444
6649
|
): Promise<boolean> {
|
|
6445
6650
|
if (ranges.length === 0) {
|
|
@@ -6496,6 +6701,13 @@ export class SharedLog<
|
|
|
6496
6701
|
ownerHash,
|
|
6497
6702
|
);
|
|
6498
6703
|
ranges = deletion.removed;
|
|
6704
|
+
if (
|
|
6705
|
+
(options?.shouldRemove && !options.shouldRemove()) ||
|
|
6706
|
+
options?.onDurableRemoveCommitted?.() === false
|
|
6707
|
+
) {
|
|
6708
|
+
await deletion.rollback();
|
|
6709
|
+
return false;
|
|
6710
|
+
}
|
|
6499
6711
|
options?.onRemoved?.(ranges);
|
|
6500
6712
|
|
|
6501
6713
|
if (ranges.length > 0) {
|
|
@@ -6581,6 +6793,7 @@ export class SharedLog<
|
|
|
6581
6793
|
timestamp?: number;
|
|
6582
6794
|
allowLegacyOrderedReplacementPairs?: boolean;
|
|
6583
6795
|
onConfirmedDurableStateChanged?: () => void;
|
|
6796
|
+
onDurableApplyCommitted?: () => boolean | void;
|
|
6584
6797
|
shouldApply?: () => boolean;
|
|
6585
6798
|
} = {},
|
|
6586
6799
|
replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
@@ -6622,6 +6835,7 @@ export class SharedLog<
|
|
|
6622
6835
|
rebalance,
|
|
6623
6836
|
allowLegacyOrderedReplacementPairs,
|
|
6624
6837
|
onConfirmedDurableStateChanged,
|
|
6838
|
+
onDurableApplyCommitted,
|
|
6625
6839
|
shouldApply,
|
|
6626
6840
|
}: {
|
|
6627
6841
|
reset?: boolean;
|
|
@@ -6630,6 +6844,7 @@ export class SharedLog<
|
|
|
6630
6844
|
timestamp?: number;
|
|
6631
6845
|
allowLegacyOrderedReplacementPairs?: boolean;
|
|
6632
6846
|
onConfirmedDurableStateChanged?: () => void;
|
|
6847
|
+
onDurableApplyCommitted?: () => boolean | void;
|
|
6633
6848
|
shouldApply?: () => boolean;
|
|
6634
6849
|
} = {},
|
|
6635
6850
|
replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
@@ -6644,6 +6859,12 @@ export class SharedLog<
|
|
|
6644
6859
|
if (shouldApply && !shouldApply()) {
|
|
6645
6860
|
return [];
|
|
6646
6861
|
}
|
|
6862
|
+
const applySuperseded = Symbol("replication-range-apply-superseded");
|
|
6863
|
+
const throwIfApplySuperseded = () => {
|
|
6864
|
+
if (shouldApply && !shouldApply()) {
|
|
6865
|
+
throw applySuperseded;
|
|
6866
|
+
}
|
|
6867
|
+
};
|
|
6647
6868
|
const fromHash = from.hashcode();
|
|
6648
6869
|
const incomingRangesById = new Map<
|
|
6649
6870
|
string,
|
|
@@ -6733,6 +6954,7 @@ export class SharedLog<
|
|
|
6733
6954
|
let isStoppedReplicating = false;
|
|
6734
6955
|
let wasReplicatorBeforeDestructiveReset = false;
|
|
6735
6956
|
let resetFailureLeaveEmitted = false;
|
|
6957
|
+
let resetDeletionRollback: (() => Promise<void>) | undefined;
|
|
6736
6958
|
const publishConfirmedResetStop = (ownerHasRanges: boolean) => {
|
|
6737
6959
|
if (ownerHasRanges || resetFailureLeaveEmitted) {
|
|
6738
6960
|
return;
|
|
@@ -6785,12 +7007,18 @@ export class SharedLog<
|
|
|
6785
7007
|
} else {
|
|
6786
7008
|
wasReplicatorBeforeDestructiveReset =
|
|
6787
7009
|
this.uniqueReplicators.has(fromHash);
|
|
7010
|
+
throwIfApplySuperseded();
|
|
6788
7011
|
const deletion = await this.deleteReplicationRangesCoherently(
|
|
6789
7012
|
deleted,
|
|
6790
7013
|
fromHash,
|
|
6791
7014
|
{ preserveOwnerMembership: ranges.length > 0 },
|
|
6792
7015
|
);
|
|
6793
7016
|
deleted = deletion.removed;
|
|
7017
|
+
resetDeletionRollback = deletion.rollback;
|
|
7018
|
+
if (shouldApply && !shouldApply()) {
|
|
7019
|
+
await deletion.rollback();
|
|
7020
|
+
return [];
|
|
7021
|
+
}
|
|
6794
7022
|
|
|
6795
7023
|
diffs = [
|
|
6796
7024
|
...deleted.map((x) => {
|
|
@@ -6986,18 +7214,26 @@ export class SharedLog<
|
|
|
6986
7214
|
};
|
|
6987
7215
|
|
|
6988
7216
|
try {
|
|
7217
|
+
throwIfApplySuperseded();
|
|
6989
7218
|
for (const diff of diffs) {
|
|
6990
7219
|
if (diff.type !== "added") {
|
|
6991
7220
|
continue;
|
|
6992
7221
|
}
|
|
7222
|
+
throwIfApplySuperseded();
|
|
6993
7223
|
appliedPositiveRanges.push(diff);
|
|
6994
7224
|
await this.replicationIndex.put(diff.range);
|
|
6995
7225
|
this.putNativeReplicationRange(diff.range);
|
|
7226
|
+
throwIfApplySuperseded();
|
|
6996
7227
|
}
|
|
6997
7228
|
if (reset && diffs.length > 0) {
|
|
6998
7229
|
await this.updateOldestTimestampFromIndex();
|
|
7230
|
+
throwIfApplySuperseded();
|
|
7231
|
+
}
|
|
7232
|
+
if (onDurableApplyCommitted?.() === false) {
|
|
7233
|
+
throw applySuperseded;
|
|
6999
7234
|
}
|
|
7000
7235
|
} catch (error) {
|
|
7236
|
+
const superseded = error === applySuperseded;
|
|
7001
7237
|
let outcomeError = error;
|
|
7002
7238
|
if (appliedPositiveRanges.length > 0) {
|
|
7003
7239
|
try {
|
|
@@ -7006,6 +7242,19 @@ export class SharedLog<
|
|
|
7006
7242
|
outcomeError = poisonFromPositiveRollback(rollbackError, error);
|
|
7007
7243
|
}
|
|
7008
7244
|
}
|
|
7245
|
+
if (superseded) {
|
|
7246
|
+
if (resetDeletionRollback) {
|
|
7247
|
+
try {
|
|
7248
|
+
await resetDeletionRollback();
|
|
7249
|
+
} catch (rollbackError) {
|
|
7250
|
+
outcomeError = rollbackError;
|
|
7251
|
+
}
|
|
7252
|
+
}
|
|
7253
|
+
if (outcomeError === applySuperseded) {
|
|
7254
|
+
return [];
|
|
7255
|
+
}
|
|
7256
|
+
throw outcomeError;
|
|
7257
|
+
}
|
|
7009
7258
|
if (reset) {
|
|
7010
7259
|
const negativeDiffs = diffs.filter((diff) => diff.type !== "added");
|
|
7011
7260
|
if (negativeDiffs.length > 0) {
|
|
@@ -8612,7 +8861,8 @@ export class SharedLog<
|
|
|
8612
8861
|
}
|
|
8613
8862
|
};
|
|
8614
8863
|
|
|
8615
|
-
const residentEntriesByHash =
|
|
8864
|
+
const residentEntriesByHash =
|
|
8865
|
+
this._coordinates._residentEntryCoordinatesByHash;
|
|
8616
8866
|
if (
|
|
8617
8867
|
(this._nativeBackbone ?? this._nativeSharedLogState) &&
|
|
8618
8868
|
residentEntriesByHash &&
|
|
@@ -8924,7 +9174,8 @@ export class SharedLog<
|
|
|
8924
9174
|
const localLeaderHashes = new Set<string>();
|
|
8925
9175
|
const unresolvedHashes = new Set(hashes);
|
|
8926
9176
|
const nativePlanner = this._nativeBackbone ?? this._nativeRangePlanner;
|
|
8927
|
-
const nativeEntryMetadata =
|
|
9177
|
+
const nativeEntryMetadata =
|
|
9178
|
+
this._coordinates.getNativeLogEntryMetadataBatch(hashes);
|
|
8928
9179
|
|
|
8929
9180
|
if (nativePlanner && !this.hasCustomFindLeaders() && nativeEntryMetadata) {
|
|
8930
9181
|
const nativeItems: Array<{
|
|
@@ -10425,7 +10676,8 @@ export class SharedLog<
|
|
|
10425
10676
|
) {
|
|
10426
10677
|
try {
|
|
10427
10678
|
this.restoreNativeBackboneDocument(nativeDocumentRollback);
|
|
10428
|
-
const flushed =
|
|
10679
|
+
const flushed =
|
|
10680
|
+
this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
10429
10681
|
if (isPromiseLike(flushed)) {
|
|
10430
10682
|
await flushed;
|
|
10431
10683
|
}
|
|
@@ -10515,11 +10767,12 @@ export class SharedLog<
|
|
|
10515
10767
|
prepared.trimmedEntries as Array<{ hash?: string }> | undefined
|
|
10516
10768
|
)?.flatMap((entry) => (entry.hash ? [entry.hash] : [])) ??
|
|
10517
10769
|
[];
|
|
10518
|
-
const coordinateRollback =
|
|
10519
|
-
|
|
10520
|
-
|
|
10521
|
-
|
|
10522
|
-
|
|
10770
|
+
const coordinateRollback =
|
|
10771
|
+
this._coordinates.snapshotResidentCoordinateEntries([
|
|
10772
|
+
...(preparedHash ? [preparedHash] : []),
|
|
10773
|
+
...preparedNext,
|
|
10774
|
+
...nativeTrimmedHashes,
|
|
10775
|
+
]);
|
|
10523
10776
|
lowerPublicationRollback = {
|
|
10524
10777
|
committedHashes: preparedHash ? [preparedHash] : [],
|
|
10525
10778
|
trimmedEntries: prepared.trimmedEntries,
|
|
@@ -10735,7 +10988,8 @@ export class SharedLog<
|
|
|
10735
10988
|
for (const document of lowerPublicationRollback?.documents ?? []) {
|
|
10736
10989
|
this.restoreNativeBackboneDocument(document);
|
|
10737
10990
|
}
|
|
10738
|
-
const flushed =
|
|
10991
|
+
const flushed =
|
|
10992
|
+
this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
10739
10993
|
if (isPromiseLike(flushed)) {
|
|
10740
10994
|
await flushed;
|
|
10741
10995
|
}
|
|
@@ -10830,7 +11084,10 @@ export class SharedLog<
|
|
|
10830
11084
|
ownershipLifecycleController,
|
|
10831
11085
|
);
|
|
10832
11086
|
const backbone = this._nativeBackbone;
|
|
10833
|
-
if (
|
|
11087
|
+
if (
|
|
11088
|
+
!backbone ||
|
|
11089
|
+
!this._coordinates.canUseNativeBackboneResidentCoordinateState()
|
|
11090
|
+
) {
|
|
10834
11091
|
return undefined;
|
|
10835
11092
|
}
|
|
10836
11093
|
if (
|
|
@@ -11035,11 +11292,12 @@ export class SharedLog<
|
|
|
11035
11292
|
const committedHash =
|
|
11036
11293
|
backboneAppend.entry.cid ?? backboneAppend.entry.hash;
|
|
11037
11294
|
const committedNext = backboneAppend.entry.next ?? next;
|
|
11038
|
-
nativeCoordinateRollback =
|
|
11039
|
-
|
|
11040
|
-
|
|
11041
|
-
|
|
11042
|
-
|
|
11295
|
+
nativeCoordinateRollback =
|
|
11296
|
+
this._coordinates.snapshotResidentCoordinateEntries([
|
|
11297
|
+
...(committedHash ? [committedHash] : []),
|
|
11298
|
+
...committedNext,
|
|
11299
|
+
...nativeTrimmedHashes,
|
|
11300
|
+
]);
|
|
11043
11301
|
await this.setNativeStrictDurableTransactionOperation(
|
|
11044
11302
|
nativeStrictTransaction,
|
|
11045
11303
|
committedHash ? [committedHash] : [],
|
|
@@ -11350,7 +11608,8 @@ export class SharedLog<
|
|
|
11350
11608
|
if (nativeDocumentRollback) {
|
|
11351
11609
|
try {
|
|
11352
11610
|
this.restoreNativeBackboneDocument(nativeDocumentRollback);
|
|
11353
|
-
const flushed =
|
|
11611
|
+
const flushed =
|
|
11612
|
+
this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
11354
11613
|
if (isPromiseLike(flushed)) {
|
|
11355
11614
|
await flushed;
|
|
11356
11615
|
}
|
|
@@ -11382,15 +11641,17 @@ export class SharedLog<
|
|
|
11382
11641
|
coordinateIndex.putSharedLogCoordinateFieldsEncodedAndDeleteHashesNoReturn ||
|
|
11383
11642
|
coordinateIndex.putSharedLogCoordinateFieldsAndDeleteHashesNoReturn;
|
|
11384
11643
|
const persisted = hasNativeCoordinatePut
|
|
11385
|
-
? this._coordinates.persistBackboneCoordinateFieldsNativeTransaction(
|
|
11386
|
-
|
|
11387
|
-
|
|
11388
|
-
|
|
11389
|
-
|
|
11390
|
-
|
|
11391
|
-
|
|
11392
|
-
|
|
11393
|
-
|
|
11644
|
+
? this._coordinates.persistBackboneCoordinateFieldsNativeTransaction(
|
|
11645
|
+
{
|
|
11646
|
+
coordinateIndex,
|
|
11647
|
+
fields: coordinateFields,
|
|
11648
|
+
hash: prepared.appendFacts.hash,
|
|
11649
|
+
deleteHashes: [],
|
|
11650
|
+
coordinates: backboneAppend.coordinate
|
|
11651
|
+
.coordinates as NumberFromType<R>[],
|
|
11652
|
+
skipGenericTransientCoordinateIndex: runtimeOnlyCoordinates,
|
|
11653
|
+
},
|
|
11654
|
+
)
|
|
11394
11655
|
: this._coordinates.persistPreparedCoordinate({
|
|
11395
11656
|
prepared: getPreparedCoordinate(),
|
|
11396
11657
|
hash: prepared.appendFacts.hash,
|
|
@@ -11444,9 +11705,10 @@ export class SharedLog<
|
|
|
11444
11705
|
if (!backboneAppend.isLeader && !delayAdaptiveRebalance) {
|
|
11445
11706
|
const leaders = backboneAppend.leaders;
|
|
11446
11707
|
if (leaders) {
|
|
11447
|
-
const pruneEntry =
|
|
11448
|
-
|
|
11449
|
-
|
|
11708
|
+
const pruneEntry =
|
|
11709
|
+
this._coordinates.materializePreparedCoordinateEntry(
|
|
11710
|
+
getPreparedCoordinate(),
|
|
11711
|
+
);
|
|
11450
11712
|
this.pruneDebouncedFnAddIfNotKeeping({
|
|
11451
11713
|
key: pruneEntry.hash,
|
|
11452
11714
|
value: { entry: pruneEntry, leaders },
|
|
@@ -12117,16 +12379,17 @@ export class SharedLog<
|
|
|
12117
12379
|
),
|
|
12118
12380
|
);
|
|
12119
12381
|
const nativeTrimmedHashes = [...nativeTrimmedHashSet];
|
|
12120
|
-
batchCoordinateRollback =
|
|
12121
|
-
|
|
12122
|
-
|
|
12123
|
-
|
|
12124
|
-
|
|
12125
|
-
|
|
12126
|
-
|
|
12127
|
-
append.
|
|
12128
|
-
|
|
12129
|
-
|
|
12382
|
+
batchCoordinateRollback =
|
|
12383
|
+
this._coordinates.snapshotResidentCoordinateEntries(
|
|
12384
|
+
committedAppends.flatMap((append) => [
|
|
12385
|
+
...((append.entry.cid ?? append.entry.hash)
|
|
12386
|
+
? [append.entry.cid ?? append.entry.hash!]
|
|
12387
|
+
: []),
|
|
12388
|
+
...append.entry.next,
|
|
12389
|
+
...(append.trimmedHashes ??
|
|
12390
|
+
append.trimmed.map((entry) => entry.hash)),
|
|
12391
|
+
]),
|
|
12392
|
+
);
|
|
12130
12393
|
await this.setNativeStrictDurableTransactionOperation(
|
|
12131
12394
|
nativeStrictTransaction,
|
|
12132
12395
|
committedCids,
|
|
@@ -12378,7 +12641,8 @@ export class SharedLog<
|
|
|
12378
12641
|
for (const document of batchDocumentRollbacks) {
|
|
12379
12642
|
this.restoreNativeBackboneDocument(document);
|
|
12380
12643
|
}
|
|
12381
|
-
const flushed =
|
|
12644
|
+
const flushed =
|
|
12645
|
+
this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
12382
12646
|
if (isPromiseLike(flushed)) {
|
|
12383
12647
|
await flushed;
|
|
12384
12648
|
}
|
|
@@ -12458,20 +12722,21 @@ export class SharedLog<
|
|
|
12458
12722
|
coordinateFields,
|
|
12459
12723
|
plannedCoordinateDeleteHashes,
|
|
12460
12724
|
});
|
|
12461
|
-
const persisted =
|
|
12462
|
-
|
|
12463
|
-
|
|
12464
|
-
|
|
12465
|
-
|
|
12466
|
-
|
|
12467
|
-
|
|
12468
|
-
|
|
12469
|
-
|
|
12470
|
-
|
|
12471
|
-
|
|
12472
|
-
|
|
12473
|
-
|
|
12474
|
-
|
|
12725
|
+
const persisted =
|
|
12726
|
+
this._coordinates.persistBackboneCoordinateFieldsNativeTransaction(
|
|
12727
|
+
{
|
|
12728
|
+
coordinateIndex: this.entryCoordinatesIndex as PutAndDeleteIndex<
|
|
12729
|
+
EntryReplicated<R>
|
|
12730
|
+
>,
|
|
12731
|
+
fields: coordinateFields,
|
|
12732
|
+
hash: facts.hash,
|
|
12733
|
+
deleteHashes: [],
|
|
12734
|
+
coordinates: backboneAppend.coordinate
|
|
12735
|
+
.coordinates as NumberFromType<R>[],
|
|
12736
|
+
skipGenericTransientCoordinateIndex: runtimeOnlyCoordinates,
|
|
12737
|
+
},
|
|
12738
|
+
ownershipLifecycleController,
|
|
12739
|
+
);
|
|
12475
12740
|
if (isPromiseLike(persisted)) {
|
|
12476
12741
|
await persisted;
|
|
12477
12742
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
@@ -13982,6 +14247,8 @@ export class SharedLog<
|
|
|
13982
14247
|
this._lastLocalAppendAt = 0;
|
|
13983
14248
|
this._announcements ??= this.createReplicationAnnouncementCoordinator();
|
|
13984
14249
|
this._announcements.resetForOpen();
|
|
14250
|
+
this._v2Receive ??= this.createReplicationInfoV2ReceiveCoordinator();
|
|
14251
|
+
this._v2Receive.resetForOpen();
|
|
13985
14252
|
this._v2Send ??= this.createReplicationInfoV2SendCoordinator();
|
|
13986
14253
|
this._v2Send.resetForOpen();
|
|
13987
14254
|
const adaptiveReplicateOptions =
|
|
@@ -14725,8 +14992,11 @@ export class SharedLog<
|
|
|
14725
14992
|
// tombstone was written. Complete that erase before the adapter can expose
|
|
14726
14993
|
// any stale coordinate or document state to this backbone.
|
|
14727
14994
|
await this._coordinates._nativeBackboneCoordinatePersistence.resumeDrop?.();
|
|
14728
|
-
await this._coordinates._nativeBackboneCoordinatePersistence.hydrate(
|
|
14729
|
-
|
|
14995
|
+
await this._coordinates._nativeBackboneCoordinatePersistence.hydrate(
|
|
14996
|
+
backbone,
|
|
14997
|
+
);
|
|
14998
|
+
this._coordinates._nativeBackboneCoordinateJournalLastFlushMs =
|
|
14999
|
+
Date.now();
|
|
14730
15000
|
this.hydrateNativeCoordinateStateFromBackbone(backbone);
|
|
14731
15001
|
return;
|
|
14732
15002
|
}
|
|
@@ -14766,7 +15036,9 @@ export class SharedLog<
|
|
|
14766
15036
|
if (!this._nativeBackbone) {
|
|
14767
15037
|
return;
|
|
14768
15038
|
}
|
|
14769
|
-
const hashes = new Set(
|
|
15039
|
+
const hashes = new Set(
|
|
15040
|
+
this._coordinates._residentEntryCoordinatesByHash?.keys() ?? [],
|
|
15041
|
+
);
|
|
14770
15042
|
const iterator = this.entryCoordinatesIndex.iterate({});
|
|
14771
15043
|
try {
|
|
14772
15044
|
for (;;) {
|
|
@@ -14836,7 +15108,10 @@ export class SharedLog<
|
|
|
14836
15108
|
coordinate.requestedReplicas,
|
|
14837
15109
|
sharedFields.hashNumber,
|
|
14838
15110
|
);
|
|
14839
|
-
this._coordinates._residentEntryCoordinatesByHash.set(
|
|
15111
|
+
this._coordinates._residentEntryCoordinatesByHash.set(
|
|
15112
|
+
sharedFields.hash,
|
|
15113
|
+
sharedFields,
|
|
15114
|
+
);
|
|
14840
15115
|
for (const value of sharedFields.coordinates) {
|
|
14841
15116
|
this.coordinateToHash.add(value, sharedFields.hash);
|
|
14842
15117
|
}
|
|
@@ -14995,7 +15270,8 @@ export class SharedLog<
|
|
|
14995
15270
|
this._coordinates._nativeBackboneCoordinatePersistence
|
|
14996
15271
|
) {
|
|
14997
15272
|
if (
|
|
14998
|
-
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15273
|
+
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15274
|
+
.durableBarrier !== true ||
|
|
14999
15275
|
typeof this._nativeBackboneCoordinatePersistenceStore
|
|
15000
15276
|
?.durableBarrier !== "function"
|
|
15001
15277
|
) {
|
|
@@ -15006,11 +15282,12 @@ export class SharedLog<
|
|
|
15006
15282
|
}
|
|
15007
15283
|
if (
|
|
15008
15284
|
this._coordinates._nativeBackboneCoordinatePersistence &&
|
|
15009
|
-
(this._coordinates._nativeBackboneCoordinatePersistence
|
|
15010
|
-
null ||
|
|
15011
|
-
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15012
|
-
null) &&
|
|
15013
|
-
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15285
|
+
(this._coordinates._nativeBackboneCoordinatePersistence
|
|
15286
|
+
.compactMaxJournalBytes != null ||
|
|
15287
|
+
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15288
|
+
.compactMaxJournalRecords != null) &&
|
|
15289
|
+
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15290
|
+
.crashSafeCompaction !== true
|
|
15014
15291
|
) {
|
|
15015
15292
|
// Durable custom adapters must explicitly advertise an atomic generation
|
|
15016
15293
|
// protocol before SharedLog permits automatic WAL compaction.
|
|
@@ -15236,7 +15513,8 @@ export class SharedLog<
|
|
|
15236
15513
|
async afterOpen(): Promise<void> {
|
|
15237
15514
|
await super.afterOpen();
|
|
15238
15515
|
const existingSubscribersPromise = this._getTopicSubscribers(this.topic);
|
|
15239
|
-
const replicationLifecycleController =
|
|
15516
|
+
const replicationLifecycleController =
|
|
15517
|
+
this._instanceLifecycle?.membershipLifecycleController;
|
|
15240
15518
|
|
|
15241
15519
|
// We do this here, because these calls requires this.closed == false
|
|
15242
15520
|
void this.pruneOfflineReplicators()
|
|
@@ -15277,7 +15555,8 @@ export class SharedLog<
|
|
|
15277
15555
|
async pruneOfflineReplicators() {
|
|
15278
15556
|
// Go through all segments and wait for replicators to become reachable;
|
|
15279
15557
|
// otherwise prune them away from the local membership view.
|
|
15280
|
-
const replicationLifecycleController =
|
|
15558
|
+
const replicationLifecycleController =
|
|
15559
|
+
this._instanceLifecycle?.membershipLifecycleController;
|
|
15281
15560
|
try {
|
|
15282
15561
|
if (
|
|
15283
15562
|
!replicationLifecycleController ||
|
|
@@ -15472,13 +15751,19 @@ export class SharedLog<
|
|
|
15472
15751
|
peerHash: string,
|
|
15473
15752
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
15474
15753
|
) {
|
|
15754
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
15755
|
+
const preserveV2Session =
|
|
15756
|
+
peerSession?.phase === "open" && peerSession.isActive();
|
|
15475
15757
|
this.cancelReplicationInfoRequests(peerHash);
|
|
15476
15758
|
this._liveness._replicatorLivenessFailures.delete(peerHash);
|
|
15477
15759
|
this._liveness._replicatorLastActivityAt.delete(peerHash);
|
|
15478
|
-
|
|
15479
|
-
|
|
15480
|
-
|
|
15481
|
-
|
|
15760
|
+
if (!preserveV2Session) {
|
|
15761
|
+
this._peerSyncCapabilities.delete(peerHash);
|
|
15762
|
+
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
15763
|
+
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
15764
|
+
this._v2Receive.clearPeer(peerHash);
|
|
15765
|
+
this._v2Send.clearPeer(peerHash);
|
|
15766
|
+
}
|
|
15482
15767
|
this.cleanupPendingIHavePeer(peerHash);
|
|
15483
15768
|
this.cleanupCheckedPrunePeer(
|
|
15484
15769
|
peerHash,
|
|
@@ -15502,8 +15787,16 @@ export class SharedLog<
|
|
|
15502
15787
|
// when they eventually reach the apply lane. Reset the sender's
|
|
15503
15788
|
// ordering watermark with the local epoch so a later arrival can be
|
|
15504
15789
|
// accepted without comparing its clock to this receiver's clock.
|
|
15505
|
-
this._peerSessions.advanceReceiveEpoch(peerHash);
|
|
15790
|
+
const receiveEpoch = this._peerSessions.advanceReceiveEpoch(peerHash);
|
|
15506
15791
|
this.latestReplicationInfoMessage.delete(peerHash);
|
|
15792
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
15793
|
+
if (peerSession?.phase === "open") {
|
|
15794
|
+
this._v2Receive.advanceRecovery({
|
|
15795
|
+
peerHash,
|
|
15796
|
+
peerSession,
|
|
15797
|
+
receiveEpoch,
|
|
15798
|
+
});
|
|
15799
|
+
}
|
|
15507
15800
|
}
|
|
15508
15801
|
|
|
15509
15802
|
private async resolveCandidatePeersForHash(
|
|
@@ -15784,7 +16077,9 @@ export class SharedLog<
|
|
|
15784
16077
|
deferredCoordinateDeleteHashes,
|
|
15785
16078
|
);
|
|
15786
16079
|
} else {
|
|
15787
|
-
this._coordinates.forgetCoordinateStateForHashes(
|
|
16080
|
+
this._coordinates.forgetCoordinateStateForHashes(
|
|
16081
|
+
deferredCoordinateDeleteHashes,
|
|
16082
|
+
);
|
|
15788
16083
|
}
|
|
15789
16084
|
return deferredCoordinateDeleteHashes;
|
|
15790
16085
|
}
|
|
@@ -15832,7 +16127,9 @@ export class SharedLog<
|
|
|
15832
16127
|
deferredCoordinateDeleteHashes,
|
|
15833
16128
|
);
|
|
15834
16129
|
} else {
|
|
15835
|
-
this._coordinates.forgetCoordinateStateForHashes(
|
|
16130
|
+
this._coordinates.forgetCoordinateStateForHashes(
|
|
16131
|
+
deferredCoordinateDeleteHashes,
|
|
16132
|
+
);
|
|
15836
16133
|
}
|
|
15837
16134
|
return deferredCoordinateDeleteHashes;
|
|
15838
16135
|
}
|
|
@@ -16395,7 +16692,9 @@ export class SharedLog<
|
|
|
16395
16692
|
this._wireSyncSession = undefined;
|
|
16396
16693
|
}
|
|
16397
16694
|
});
|
|
16398
|
-
await capture(() =>
|
|
16695
|
+
await capture(() =>
|
|
16696
|
+
this._coordinates.closeNativeBackboneCoordinatePersistence(),
|
|
16697
|
+
);
|
|
16399
16698
|
await capture(() => this.syncronizer?.close());
|
|
16400
16699
|
|
|
16401
16700
|
captureSync(() => {
|
|
@@ -16490,6 +16789,7 @@ export class SharedLog<
|
|
|
16490
16789
|
this._peerSyncCapabilities?.clear();
|
|
16491
16790
|
this._peerSyncCapabilitySessions?.clear();
|
|
16492
16791
|
this._peerSyncCapabilityTimestamps?.clear();
|
|
16792
|
+
this._v2Receive?.clearForClose();
|
|
16493
16793
|
this._v2Send?.clearForClose();
|
|
16494
16794
|
this._liveRawGossipBatches?.clear();
|
|
16495
16795
|
this._nativeSharedLogState?.clearGidPeers();
|
|
@@ -16563,6 +16863,7 @@ export class SharedLog<
|
|
|
16563
16863
|
this.preventParentAttachments();
|
|
16564
16864
|
this.stopRepairLifecycle();
|
|
16565
16865
|
this._instanceLifecycle?.beginTerminal("close");
|
|
16866
|
+
this._v2Receive?.clearForClose();
|
|
16566
16867
|
const replicationRangeTerminalFence =
|
|
16567
16868
|
this.acquireReplicationRangeMutationTerminalFence();
|
|
16568
16869
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
@@ -16625,10 +16926,12 @@ export class SharedLog<
|
|
|
16625
16926
|
try {
|
|
16626
16927
|
const reset = new AllReplicatingSegmentsMessage({ segments: [] });
|
|
16627
16928
|
await Promise.all([
|
|
16628
|
-
this.rpc
|
|
16629
|
-
|
|
16630
|
-
|
|
16631
|
-
|
|
16929
|
+
this.rpc
|
|
16930
|
+
.send(reset, {
|
|
16931
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
16932
|
+
signal: abort.signal,
|
|
16933
|
+
})
|
|
16934
|
+
.catch(() => {}),
|
|
16632
16935
|
this._v2Send.sendTerminalReset(abort.signal),
|
|
16633
16936
|
]);
|
|
16634
16937
|
} finally {
|
|
@@ -16688,7 +16991,8 @@ export class SharedLog<
|
|
|
16688
16991
|
}
|
|
16689
16992
|
this.throwIfCheckedPruneRemoveBlocksLocalOperation("drop");
|
|
16690
16993
|
this.ensureNativeDurabilityRuntimeState();
|
|
16691
|
-
const nativePersistence =
|
|
16994
|
+
const nativePersistence =
|
|
16995
|
+
this._coordinates._nativeBackboneCoordinatePersistence;
|
|
16692
16996
|
if (
|
|
16693
16997
|
nativePersistence &&
|
|
16694
16998
|
(typeof nativePersistence.drop !== "function" ||
|
|
@@ -16706,6 +17010,7 @@ export class SharedLog<
|
|
|
16706
17010
|
this.preventParentAttachments();
|
|
16707
17011
|
this.stopRepairLifecycle();
|
|
16708
17012
|
this._instanceLifecycle?.beginTerminal("drop");
|
|
17013
|
+
this._v2Receive?.clearForClose();
|
|
16709
17014
|
const replicationRangeTerminalFence =
|
|
16710
17015
|
this.acquireReplicationRangeMutationTerminalFence();
|
|
16711
17016
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
@@ -16751,10 +17056,12 @@ export class SharedLog<
|
|
|
16751
17056
|
try {
|
|
16752
17057
|
const reset = new AllReplicatingSegmentsMessage({ segments: [] });
|
|
16753
17058
|
await Promise.all([
|
|
16754
|
-
this.rpc
|
|
16755
|
-
|
|
16756
|
-
|
|
16757
|
-
|
|
17059
|
+
this.rpc
|
|
17060
|
+
.send(reset, {
|
|
17061
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
17062
|
+
signal: abort.signal,
|
|
17063
|
+
})
|
|
17064
|
+
.catch(() => {}),
|
|
16758
17065
|
this._v2Send.sendTerminalReset(abort.signal),
|
|
16759
17066
|
]);
|
|
16760
17067
|
} finally {
|
|
@@ -17273,13 +17580,6 @@ export class SharedLog<
|
|
|
17273
17580
|
msg: TransportMessage,
|
|
17274
17581
|
context: RequestContext,
|
|
17275
17582
|
): Promise<void> {
|
|
17276
|
-
// Decode-first only: capability advertisement is not permission to use
|
|
17277
|
-
// V2. Drop every unsolicited V2 envelope before even entering the shared
|
|
17278
|
-
// receive envelope, so durable poison checks, leases, liveness, ordering
|
|
17279
|
-
// watermarks, replication ranges and cleanup side effects cannot observe it.
|
|
17280
|
-
if (isReplicationInfoV2Message(msg)) {
|
|
17281
|
-
return;
|
|
17282
|
-
}
|
|
17283
17583
|
const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
|
|
17284
17584
|
? msg
|
|
17285
17585
|
: undefined;
|
|
@@ -17332,19 +17632,28 @@ export class SharedLog<
|
|
|
17332
17632
|
}
|
|
17333
17633
|
if (
|
|
17334
17634
|
msg instanceof AllReplicatingSegmentsMessage ||
|
|
17335
|
-
msg instanceof AddedReplicationSegmentMessage
|
|
17635
|
+
msg instanceof AddedReplicationSegmentMessage ||
|
|
17636
|
+
msg instanceof FullReplicationInfoV2Message ||
|
|
17637
|
+
msg instanceof AddedReplicationInfoV2Message
|
|
17336
17638
|
) {
|
|
17337
17639
|
// Bound decoded untrusted vectors before per-peer/global mutation
|
|
17338
17640
|
// queues, trusted-replicator authorization, or liveness side effects.
|
|
17339
17641
|
this.validateReplicationRangeAnnouncement(msg.segments);
|
|
17340
|
-
} else if (
|
|
17642
|
+
} else if (
|
|
17643
|
+
msg instanceof StoppedReplicating ||
|
|
17644
|
+
msg instanceof StoppedReplicationInfoV2Message
|
|
17645
|
+
) {
|
|
17341
17646
|
// Bound the raw decoded vector before deduplication can hide the
|
|
17342
17647
|
// allocation cost, and before liveness or apply-queue side effects.
|
|
17343
17648
|
this.validateStoppedReplicationAnnouncement(msg.segmentIds);
|
|
17344
17649
|
}
|
|
17345
17650
|
if (
|
|
17346
17651
|
!context.from.equals(this.node.identity.publicKey) &&
|
|
17347
|
-
!(msg instanceof RequestReplicationInfoV2Message)
|
|
17652
|
+
!(msg instanceof RequestReplicationInfoV2Message) &&
|
|
17653
|
+
!isReplicationInfoV2Message(msg) &&
|
|
17654
|
+
!(msg instanceof AllReplicatingSegmentsMessage) &&
|
|
17655
|
+
!(msg instanceof AddedReplicationSegmentMessage) &&
|
|
17656
|
+
!(msg instanceof StoppedReplicating)
|
|
17348
17657
|
) {
|
|
17349
17658
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
17350
17659
|
}
|
|
@@ -18390,11 +18699,14 @@ export class SharedLog<
|
|
|
18390
18699
|
);
|
|
18391
18700
|
}
|
|
18392
18701
|
const reusableCoordinatePlans =
|
|
18393
|
-
this._coordinates.createReusableReceiveCoordinatePlans(
|
|
18394
|
-
|
|
18395
|
-
|
|
18396
|
-
|
|
18397
|
-
|
|
18702
|
+
this._coordinates.createReusableReceiveCoordinatePlans(
|
|
18703
|
+
receiveGroups,
|
|
18704
|
+
{
|
|
18705
|
+
decodedReplicaCounts: receiveReplicaCounts,
|
|
18706
|
+
allowRoleAgeZeroPlans:
|
|
18707
|
+
immediateReplicatingLeaderPlans !== undefined,
|
|
18708
|
+
},
|
|
18709
|
+
) as Map<string, ReusableReceiveCoordinatePlan<R>>;
|
|
18398
18710
|
if (syncProfile) {
|
|
18399
18711
|
emitSyncProfileDuration(syncProfile, joinPlanStartedAt, {
|
|
18400
18712
|
name: "sharedLog.receive.joinPlan",
|
|
@@ -18789,9 +19101,7 @@ export class SharedLog<
|
|
|
18789
19101
|
}
|
|
18790
19102
|
const checkedPruneStartedAt = syncProfileStart(syncProfile);
|
|
18791
19103
|
const ownershipChangedDuringReceive =
|
|
18792
|
-
!this.isReceiveOwnershipSnapshotStable(
|
|
18793
|
-
receiveOwnershipRevision,
|
|
18794
|
-
);
|
|
19104
|
+
!this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision);
|
|
18795
19105
|
if (ownershipChangedDuringReceive) {
|
|
18796
19106
|
const freshAuditRevision =
|
|
18797
19107
|
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
@@ -18949,6 +19259,7 @@ export class SharedLog<
|
|
|
18949
19259
|
fromHash: receiveFromHash,
|
|
18950
19260
|
session: receiveSession,
|
|
18951
19261
|
lifecycleController: receiveReplicationLifecycleController,
|
|
19262
|
+
ownershipLifecycleController: receiveOwnershipLifecycleController,
|
|
18952
19263
|
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
18953
19264
|
syncProfile,
|
|
18954
19265
|
lease: peerReceiveLease,
|
|
@@ -18972,6 +19283,12 @@ export class SharedLog<
|
|
|
18972
19283
|
this.markEntriesKnownByPeer(msg.hashes, context.from.hashcode());
|
|
18973
19284
|
this.clearRepairFrontierHashes(context.from.hashcode(), msg.hashes);
|
|
18974
19285
|
return;
|
|
19286
|
+
} else if (isReplicationInfoV2Message(msg)) {
|
|
19287
|
+
await this.handleReplicationInfoV2Announcement(
|
|
19288
|
+
msg,
|
|
19289
|
+
laneRequestContext,
|
|
19290
|
+
lane,
|
|
19291
|
+
);
|
|
18975
19292
|
} else if (msg instanceof SyncCapabilitiesMessage) {
|
|
18976
19293
|
if (!context.from.equals(this.node.identity.publicKey)) {
|
|
18977
19294
|
const capabilityTransportSession = context.message?.header?.session;
|
|
@@ -18993,13 +19310,19 @@ export class SharedLog<
|
|
|
18993
19310
|
timestamp: capabilityTimestamp,
|
|
18994
19311
|
openingSession: receiveSession!,
|
|
18995
19312
|
});
|
|
18996
|
-
} else
|
|
19313
|
+
} else if (
|
|
18997
19314
|
this.observePeerSyncCapabilities({
|
|
18998
19315
|
peerHash: receiveFromHash,
|
|
18999
19316
|
capabilities: msg.capabilities,
|
|
19000
19317
|
transportSession: capabilityTransportSession,
|
|
19001
19318
|
timestamp: capabilityTimestamp,
|
|
19002
|
-
})
|
|
19319
|
+
}) &&
|
|
19320
|
+
receiveSession?.phase === "open"
|
|
19321
|
+
) {
|
|
19322
|
+
this.promoteReplicationInfoV2ReceiveCapability(
|
|
19323
|
+
context.from,
|
|
19324
|
+
receiveSession,
|
|
19325
|
+
);
|
|
19003
19326
|
}
|
|
19004
19327
|
}
|
|
19005
19328
|
return;
|
|
@@ -19021,6 +19344,8 @@ export class SharedLog<
|
|
|
19021
19344
|
from: context.from,
|
|
19022
19345
|
peerSession: receiveSession,
|
|
19023
19346
|
receiverTransportSession: context.message.header.session,
|
|
19347
|
+
capabilityTimestamp:
|
|
19348
|
+
this._peerSyncCapabilityTimestamps.get(receiveFromHash)!,
|
|
19024
19349
|
requestTimestamp: context.message.header.timestamp,
|
|
19025
19350
|
})
|
|
19026
19351
|
) {
|
|
@@ -19164,9 +19489,7 @@ export class SharedLog<
|
|
|
19164
19489
|
msg.requests,
|
|
19165
19490
|
);
|
|
19166
19491
|
if (
|
|
19167
|
-
!this.isReplicationLifecycleActive(
|
|
19168
|
-
receiveReplicationLifecycleController,
|
|
19169
|
-
)
|
|
19492
|
+
!this.isReplicationLifecycleActive(receiveReplicationLifecycleController)
|
|
19170
19493
|
) {
|
|
19171
19494
|
return;
|
|
19172
19495
|
}
|
|
@@ -19225,9 +19548,7 @@ export class SharedLog<
|
|
|
19225
19548
|
let entry: Entry<T> | undefined;
|
|
19226
19549
|
try {
|
|
19227
19550
|
if (await this.log.blocks.has(request.hash)) {
|
|
19228
|
-
entry = (await this.log.get(request.hash)) as
|
|
19229
|
-
| Entry<T>
|
|
19230
|
-
| undefined;
|
|
19551
|
+
entry = (await this.log.get(request.hash)) as Entry<T> | undefined;
|
|
19231
19552
|
}
|
|
19232
19553
|
} catch {
|
|
19233
19554
|
// The normal entry-admission hook will retry this path.
|
|
@@ -19268,16 +19589,11 @@ export class SharedLog<
|
|
|
19268
19589
|
}
|
|
19269
19590
|
const responseTasks: Promise<void>[] = [];
|
|
19270
19591
|
for (const request of msg.requests) {
|
|
19271
|
-
const pendingDelete = this._checkedPrune.getPendingDelete(
|
|
19272
|
-
request.hash,
|
|
19273
|
-
);
|
|
19592
|
+
const pendingDelete = this._checkedPrune.getPendingDelete(request.hash);
|
|
19274
19593
|
if (pendingDelete) {
|
|
19275
19594
|
responseTasks.push(
|
|
19276
19595
|
Promise.resolve(
|
|
19277
|
-
pendingDelete.resolve(
|
|
19278
|
-
context.from.hashcode(),
|
|
19279
|
-
request.requestId,
|
|
19280
|
-
),
|
|
19596
|
+
pendingDelete.resolve(context.from.hashcode(), request.requestId),
|
|
19281
19597
|
),
|
|
19282
19598
|
);
|
|
19283
19599
|
}
|
|
@@ -19397,6 +19713,191 @@ export class SharedLog<
|
|
|
19397
19713
|
}
|
|
19398
19714
|
}
|
|
19399
19715
|
|
|
19716
|
+
private async handleReplicationInfoV2Announcement(
|
|
19717
|
+
msg: ReplicationInfoV2Message,
|
|
19718
|
+
context: ReceiveRequestContext,
|
|
19719
|
+
lane: ReceiveLaneContext,
|
|
19720
|
+
): Promise<void> {
|
|
19721
|
+
const from = context.from;
|
|
19722
|
+
const fromHash = lane.fromHash;
|
|
19723
|
+
const receiveSession = lane.session;
|
|
19724
|
+
if (
|
|
19725
|
+
from.equals(this.node.identity.publicKey) ||
|
|
19726
|
+
receiveSession === null ||
|
|
19727
|
+
receiveSession.phase !== "open"
|
|
19728
|
+
) {
|
|
19729
|
+
return;
|
|
19730
|
+
}
|
|
19731
|
+
|
|
19732
|
+
// Authenticate and reserve before the per-peer lane. One reservation per
|
|
19733
|
+
// peer bounds decoded-frame retention while another mutation is parked.
|
|
19734
|
+
const receiveState = this._v2Receive._receiveStates.get(fromHash);
|
|
19735
|
+
if (
|
|
19736
|
+
!receiveState ||
|
|
19737
|
+
receiveState.peerSession !== receiveSession ||
|
|
19738
|
+
receiveState.senderTransportSession !== context.message.header.session
|
|
19739
|
+
) {
|
|
19740
|
+
return;
|
|
19741
|
+
}
|
|
19742
|
+
const admission = this._v2Receive.reserve(msg, {
|
|
19743
|
+
from,
|
|
19744
|
+
peerSession: receiveSession,
|
|
19745
|
+
receiveEpoch: lane.receiveEpoch,
|
|
19746
|
+
senderTransportSession: context.message.header.session,
|
|
19747
|
+
transportTimestamp: context.message.header.timestamp,
|
|
19748
|
+
});
|
|
19749
|
+
if (!admission) {
|
|
19750
|
+
return;
|
|
19751
|
+
}
|
|
19752
|
+
|
|
19753
|
+
lane.lease.release();
|
|
19754
|
+
try {
|
|
19755
|
+
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
19756
|
+
const hostGate = () =>
|
|
19757
|
+
this._instanceLifecycle!.isMembershipActiveFor(
|
|
19758
|
+
lane.lifecycleController,
|
|
19759
|
+
) &&
|
|
19760
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController) &&
|
|
19761
|
+
this._peerSessions.isCurrent(fromHash, receiveSession) &&
|
|
19762
|
+
receiveSession.phase === "open" &&
|
|
19763
|
+
this._peerSessions.isReceiveEpochCurrent(
|
|
19764
|
+
fromHash,
|
|
19765
|
+
lane.receiveEpoch,
|
|
19766
|
+
) &&
|
|
19767
|
+
!this._peerSessions.isReplicationInfoBlocked(fromHash) &&
|
|
19768
|
+
this._peerSessions.isReceiveCleanupGateOpen(fromHash);
|
|
19769
|
+
if (!hostGate()) {
|
|
19770
|
+
return;
|
|
19771
|
+
}
|
|
19772
|
+
const exactGate = () =>
|
|
19773
|
+
hostGate() && this._v2Receive.isAdmissionCurrent(admission);
|
|
19774
|
+
let durableCommitted = false;
|
|
19775
|
+
|
|
19776
|
+
try {
|
|
19777
|
+
if (
|
|
19778
|
+
msg instanceof FullReplicationInfoV2Message ||
|
|
19779
|
+
msg instanceof AddedReplicationInfoV2Message
|
|
19780
|
+
) {
|
|
19781
|
+
let mutationGateChecked = false;
|
|
19782
|
+
let mutationGateAdmitted = false;
|
|
19783
|
+
const result = await this.addReplicationRange(
|
|
19784
|
+
msg.segments.map((segment) =>
|
|
19785
|
+
segment.toReplicationRangeIndexable(from),
|
|
19786
|
+
),
|
|
19787
|
+
from,
|
|
19788
|
+
{
|
|
19789
|
+
reset: msg instanceof FullReplicationInfoV2Message,
|
|
19790
|
+
checkDuplicates: true,
|
|
19791
|
+
timestamp: Number(context.message.header.timestamp),
|
|
19792
|
+
allowLegacyOrderedReplacementPairs:
|
|
19793
|
+
msg instanceof AddedReplicationInfoV2Message,
|
|
19794
|
+
shouldApply: () => {
|
|
19795
|
+
mutationGateChecked = true;
|
|
19796
|
+
mutationGateAdmitted = exactGate();
|
|
19797
|
+
return mutationGateAdmitted;
|
|
19798
|
+
},
|
|
19799
|
+
onDurableApplyCommitted: () => {
|
|
19800
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
19801
|
+
return false;
|
|
19802
|
+
}
|
|
19803
|
+
durableCommitted = true;
|
|
19804
|
+
return true;
|
|
19805
|
+
},
|
|
19806
|
+
},
|
|
19807
|
+
lane.ownershipLifecycleController,
|
|
19808
|
+
);
|
|
19809
|
+
if (
|
|
19810
|
+
result === undefined ||
|
|
19811
|
+
!mutationGateChecked ||
|
|
19812
|
+
!mutationGateAdmitted ||
|
|
19813
|
+
!durableCommitted ||
|
|
19814
|
+
!exactGate()
|
|
19815
|
+
) {
|
|
19816
|
+
return;
|
|
19817
|
+
}
|
|
19818
|
+
} else {
|
|
19819
|
+
if (!exactGate()) {
|
|
19820
|
+
return;
|
|
19821
|
+
}
|
|
19822
|
+
const rangesToRemove =
|
|
19823
|
+
await this.resolveReplicationRangesFromIdsAndKey(
|
|
19824
|
+
msg.segmentIds,
|
|
19825
|
+
from,
|
|
19826
|
+
);
|
|
19827
|
+
if (!exactGate()) {
|
|
19828
|
+
return;
|
|
19829
|
+
}
|
|
19830
|
+
let mutationGateAdmitted = true;
|
|
19831
|
+
const removedRanges: ReplicationRangeIndexable<R>[] = [];
|
|
19832
|
+
const removed = await this.removeReplicationRanges(
|
|
19833
|
+
rangesToRemove,
|
|
19834
|
+
from,
|
|
19835
|
+
{
|
|
19836
|
+
shouldRemove: () => {
|
|
19837
|
+
mutationGateAdmitted = exactGate();
|
|
19838
|
+
return mutationGateAdmitted;
|
|
19839
|
+
},
|
|
19840
|
+
onDurableRemoveCommitted: () => {
|
|
19841
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
19842
|
+
return false;
|
|
19843
|
+
}
|
|
19844
|
+
durableCommitted = true;
|
|
19845
|
+
return true;
|
|
19846
|
+
},
|
|
19847
|
+
onRemoved: (ranges) => removedRanges.push(...ranges),
|
|
19848
|
+
},
|
|
19849
|
+
lane.ownershipLifecycleController,
|
|
19850
|
+
);
|
|
19851
|
+
if (!durableCommitted) {
|
|
19852
|
+
if (
|
|
19853
|
+
!mutationGateAdmitted ||
|
|
19854
|
+
!exactGate() ||
|
|
19855
|
+
removed ||
|
|
19856
|
+
!this._v2Receive.commit(admission)
|
|
19857
|
+
) {
|
|
19858
|
+
return;
|
|
19859
|
+
}
|
|
19860
|
+
durableCommitted = true;
|
|
19861
|
+
}
|
|
19862
|
+
if (
|
|
19863
|
+
this._instanceLifecycle!.isMembershipActiveFor(
|
|
19864
|
+
lane.lifecycleController,
|
|
19865
|
+
) &&
|
|
19866
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController)
|
|
19867
|
+
) {
|
|
19868
|
+
const timestamp = BigInt(Date.now());
|
|
19869
|
+
for (const range of removedRanges) {
|
|
19870
|
+
this.replicationChangeDebounceFn.add({
|
|
19871
|
+
range,
|
|
19872
|
+
type: "removed",
|
|
19873
|
+
timestamp,
|
|
19874
|
+
});
|
|
19875
|
+
}
|
|
19876
|
+
}
|
|
19877
|
+
}
|
|
19878
|
+
} catch (error) {
|
|
19879
|
+
this._v2Receive.requireFullAfterFailure(admission);
|
|
19880
|
+
throw error;
|
|
19881
|
+
}
|
|
19882
|
+
|
|
19883
|
+
if (!durableCommitted) {
|
|
19884
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
19885
|
+
return;
|
|
19886
|
+
}
|
|
19887
|
+
}
|
|
19888
|
+
if (!hostGate()) {
|
|
19889
|
+
return;
|
|
19890
|
+
}
|
|
19891
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
19892
|
+
if (msg instanceof FullReplicationInfoV2Message) {
|
|
19893
|
+
this.cancelReplicationInfoRequests(fromHash);
|
|
19894
|
+
}
|
|
19895
|
+
});
|
|
19896
|
+
} finally {
|
|
19897
|
+
this._v2Receive.release(admission);
|
|
19898
|
+
}
|
|
19899
|
+
}
|
|
19900
|
+
|
|
19400
19901
|
private async handleReplicationInfoAnnouncement(
|
|
19401
19902
|
msg: AllReplicatingSegmentsMessage | AddedReplicationSegmentMessage,
|
|
19402
19903
|
context: ReceiveRequestContext,
|
|
@@ -19420,6 +19921,19 @@ export class SharedLog<
|
|
|
19420
19921
|
// (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
|
|
19421
19922
|
const from = context.from!;
|
|
19422
19923
|
const fromHash = from.hashcode();
|
|
19924
|
+
if (this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
19925
|
+
if (receiveSession) {
|
|
19926
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
19927
|
+
peerHash: fromHash,
|
|
19928
|
+
peerSession: receiveSession,
|
|
19929
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
19930
|
+
senderTransportSession: context.message.header.session,
|
|
19931
|
+
transportTimestamp: context.message.header.timestamp,
|
|
19932
|
+
message: msg,
|
|
19933
|
+
});
|
|
19934
|
+
}
|
|
19935
|
+
return;
|
|
19936
|
+
}
|
|
19423
19937
|
// Pre-lane gate: lifecycle -> receive-epoch -> blocked, exactly the
|
|
19424
19938
|
// legacy order. isMembershipActiveFor is the unit-pinned fold of
|
|
19425
19939
|
// isReplicationLifecycleActive; isReceiveEpochCurrent is the
|
|
@@ -19458,6 +19972,17 @@ export class SharedLog<
|
|
|
19458
19972
|
) {
|
|
19459
19973
|
return;
|
|
19460
19974
|
}
|
|
19975
|
+
if (receiveSession && this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
19976
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
19977
|
+
peerHash: fromHash,
|
|
19978
|
+
peerSession: receiveSession,
|
|
19979
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
19980
|
+
senderTransportSession: context.message.header.session,
|
|
19981
|
+
transportTimestamp: context.message.header.timestamp,
|
|
19982
|
+
message: msg,
|
|
19983
|
+
});
|
|
19984
|
+
return;
|
|
19985
|
+
}
|
|
19461
19986
|
|
|
19462
19987
|
// Process in-order to avoid races where repeated reset messages arrive
|
|
19463
19988
|
// concurrently and trigger spurious "added" diffs / rebalancing.
|
|
@@ -19467,14 +19992,13 @@ export class SharedLog<
|
|
|
19467
19992
|
}
|
|
19468
19993
|
|
|
19469
19994
|
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
19470
|
-
this._liveness._replicatorLivenessFailures.delete(fromHash);
|
|
19471
19995
|
|
|
19472
19996
|
if (this.closed) {
|
|
19473
19997
|
return;
|
|
19474
19998
|
}
|
|
19475
19999
|
|
|
19476
20000
|
const reset = msg instanceof AllReplicatingSegmentsMessage;
|
|
19477
|
-
await this.addReplicationRange(
|
|
20001
|
+
const result = await this.addReplicationRange(
|
|
19478
20002
|
replicationInfoMessage.segments.map((x) =>
|
|
19479
20003
|
x.toReplicationRangeIndexable(from),
|
|
19480
20004
|
),
|
|
@@ -19487,6 +20011,10 @@ export class SharedLog<
|
|
|
19487
20011
|
msg instanceof AddedReplicationSegmentMessage,
|
|
19488
20012
|
},
|
|
19489
20013
|
);
|
|
20014
|
+
if (result === undefined) {
|
|
20015
|
+
return;
|
|
20016
|
+
}
|
|
20017
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
19490
20018
|
|
|
19491
20019
|
// If the peer reports any replication segments, stop re-requesting.
|
|
19492
20020
|
// (Empty reports can be transient during startup.)
|
|
@@ -19522,6 +20050,19 @@ export class SharedLog<
|
|
|
19522
20050
|
return;
|
|
19523
20051
|
}
|
|
19524
20052
|
const fromHash = from.hashcode();
|
|
20053
|
+
if (this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
20054
|
+
if (receiveSession) {
|
|
20055
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
20056
|
+
peerHash: fromHash,
|
|
20057
|
+
peerSession: receiveSession,
|
|
20058
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
20059
|
+
senderTransportSession: context.message.header.session,
|
|
20060
|
+
transportTimestamp: context.message.header.timestamp,
|
|
20061
|
+
message: msg,
|
|
20062
|
+
});
|
|
20063
|
+
}
|
|
20064
|
+
return;
|
|
20065
|
+
}
|
|
19525
20066
|
// Same pre-lane gate shape as Added/All above (and the same
|
|
19526
20067
|
// intentional absence of a subscription-epoch term).
|
|
19527
20068
|
if (
|
|
@@ -19552,22 +20093,34 @@ export class SharedLog<
|
|
|
19552
20093
|
) {
|
|
19553
20094
|
return;
|
|
19554
20095
|
}
|
|
20096
|
+
if (receiveSession && this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
20097
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
20098
|
+
peerHash: fromHash,
|
|
20099
|
+
peerSession: receiveSession,
|
|
20100
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
20101
|
+
senderTransportSession: context.message.header.session,
|
|
20102
|
+
transportTimestamp: context.message.header.timestamp,
|
|
20103
|
+
message: msg,
|
|
20104
|
+
});
|
|
20105
|
+
return;
|
|
20106
|
+
}
|
|
19555
20107
|
|
|
19556
|
-
const previousTimestamp =
|
|
19557
|
-
this.latestReplicationInfoMessage.get(fromHash);
|
|
20108
|
+
const previousTimestamp = this.latestReplicationInfoMessage.get(fromHash);
|
|
19558
20109
|
if (previousTimestamp && previousTimestamp > messageTimestamp) {
|
|
19559
20110
|
return;
|
|
19560
20111
|
}
|
|
19561
20112
|
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
19562
|
-
this._liveness._replicatorLivenessFailures.delete(fromHash);
|
|
19563
20113
|
if (this.closed) {
|
|
19564
20114
|
return;
|
|
19565
20115
|
}
|
|
19566
20116
|
|
|
19567
|
-
const rangesToRemove =
|
|
19568
|
-
|
|
20117
|
+
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
|
|
20118
|
+
segmentIds,
|
|
20119
|
+
from,
|
|
20120
|
+
);
|
|
19569
20121
|
|
|
19570
20122
|
await this.removeReplicationRanges(rangesToRemove, from);
|
|
20123
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
19571
20124
|
const timestamp = BigInt(+new Date());
|
|
19572
20125
|
for (const range of rangesToRemove) {
|
|
19573
20126
|
this.replicationChangeDebounceFn.add({
|
|
@@ -20369,10 +20922,9 @@ export class SharedLog<
|
|
|
20369
20922
|
entry: ShallowOrFullEntry<any> | EntryReplicated<R> | NumberFromType<R>,
|
|
20370
20923
|
minReplicas: number,
|
|
20371
20924
|
): Promise<NumberFromType<R>[]> {
|
|
20372
|
-
return this._coordinates.createCoordinates(
|
|
20373
|
-
|
|
20374
|
-
|
|
20375
|
-
) as Promise<NumberFromType<R>[]>;
|
|
20925
|
+
return this._coordinates.createCoordinates(entry, minReplicas) as Promise<
|
|
20926
|
+
NumberFromType<R>[]
|
|
20927
|
+
>;
|
|
20376
20928
|
}
|
|
20377
20929
|
|
|
20378
20930
|
async getDefaultMinRoleAge(): Promise<number> {
|
|
@@ -20736,8 +21288,9 @@ export class SharedLog<
|
|
|
20736
21288
|
this.createNativeLeaderOptions(context, firstItem.options),
|
|
20737
21289
|
);
|
|
20738
21290
|
const plans: EntryLeaderPlan<R>[] = [];
|
|
20739
|
-
const persistItems: Parameters<
|
|
20740
|
-
|
|
21291
|
+
const persistItems: Parameters<
|
|
21292
|
+
typeof this._coordinates.persistCoordinatesBatch
|
|
21293
|
+
>[0] = [];
|
|
20741
21294
|
for (let i = 0; i < itemArray.length; i++) {
|
|
20742
21295
|
const item = itemArray[i]!;
|
|
20743
21296
|
const nativePlan = nativePlans[i]!;
|
|
@@ -20799,8 +21352,9 @@ export class SharedLog<
|
|
|
20799
21352
|
);
|
|
20800
21353
|
const selfHash = this.node.identity.publicKey.hashcode();
|
|
20801
21354
|
const plans: EntryLeaderPlan<R>[] = [];
|
|
20802
|
-
const persistItems: Parameters<
|
|
20803
|
-
|
|
21355
|
+
const persistItems: Parameters<
|
|
21356
|
+
typeof this._coordinates.persistCoordinatesBatch
|
|
21357
|
+
>[0] = [];
|
|
20804
21358
|
for (let i = 0; i < itemArray.length; i++) {
|
|
20805
21359
|
const item = itemArray[i]!;
|
|
20806
21360
|
const nativePlan = nativePlans[i]!;
|
|
@@ -22411,6 +22965,8 @@ export class SharedLog<
|
|
|
22411
22965
|
ownerHash: string,
|
|
22412
22966
|
options?: { preserveOwnerMembership?: boolean },
|
|
22413
22967
|
): Promise<ReplicationRangeDeletionOutcome<R>> {
|
|
22968
|
+
const wasReplicator = this.uniqueReplicators.has(ownerHash);
|
|
22969
|
+
const joinWasEmitted = this._replicatorJoinEmitted.has(ownerHash);
|
|
22414
22970
|
if (ranges.length === 0) {
|
|
22415
22971
|
const ownerHasRanges =
|
|
22416
22972
|
(await this.replicationIndex.count({ query: { hash: ownerHash } })) > 0;
|
|
@@ -22422,14 +22978,24 @@ export class SharedLog<
|
|
|
22422
22978
|
removed: [],
|
|
22423
22979
|
retained: [],
|
|
22424
22980
|
ownerHasRanges,
|
|
22981
|
+
rollback: async () => {
|
|
22982
|
+
if (wasReplicator) {
|
|
22983
|
+
this.uniqueReplicators.add(ownerHash);
|
|
22984
|
+
} else {
|
|
22985
|
+
this.uniqueReplicators.delete(ownerHash);
|
|
22986
|
+
}
|
|
22987
|
+
if (joinWasEmitted) {
|
|
22988
|
+
this._replicatorJoinEmitted.add(ownerHash);
|
|
22989
|
+
} else {
|
|
22990
|
+
this._replicatorJoinEmitted.delete(ownerHash);
|
|
22991
|
+
}
|
|
22992
|
+
},
|
|
22425
22993
|
};
|
|
22426
22994
|
}
|
|
22427
22995
|
|
|
22428
22996
|
const uniqueRanges = [
|
|
22429
22997
|
...new Map(ranges.map((range) => [range.idString, range])).values(),
|
|
22430
22998
|
];
|
|
22431
|
-
const wasReplicator = this.uniqueReplicators.has(ownerHash);
|
|
22432
|
-
const joinWasEmitted = this._replicatorJoinEmitted.has(ownerHash);
|
|
22433
22999
|
type Snapshot = {
|
|
22434
23000
|
range: ReplicationRangeIndexable<R>;
|
|
22435
23001
|
pending?: PendingMaturityRecord<R>;
|
|
@@ -22663,10 +23229,80 @@ export class SharedLog<
|
|
|
22663
23229
|
);
|
|
22664
23230
|
this.poisonReplicationOwnership(outcomeError);
|
|
22665
23231
|
}
|
|
23232
|
+
let rollbackStarted = false;
|
|
23233
|
+
const rollback = async () => {
|
|
23234
|
+
if (rollbackStarted) {
|
|
23235
|
+
return;
|
|
23236
|
+
}
|
|
23237
|
+
rollbackStarted = true;
|
|
23238
|
+
const rollbackErrors: unknown[] = [];
|
|
23239
|
+
const removedIds = new Set(removed.map((range) => range.idString));
|
|
23240
|
+
for (const snapshot of snapshots) {
|
|
23241
|
+
if (!removedIds.has(snapshot.range.idString)) {
|
|
23242
|
+
continue;
|
|
23243
|
+
}
|
|
23244
|
+
try {
|
|
23245
|
+
await this.replicationIndex.put(snapshot.range);
|
|
23246
|
+
this.putNativeReplicationRange(snapshot.range);
|
|
23247
|
+
} catch (error) {
|
|
23248
|
+
rollbackErrors.push(error);
|
|
23249
|
+
}
|
|
23250
|
+
}
|
|
23251
|
+
for (const snapshot of snapshots) {
|
|
23252
|
+
if (
|
|
23253
|
+
!removedIds.has(snapshot.range.idString) ||
|
|
23254
|
+
!snapshot.pending ||
|
|
23255
|
+
this.pendingMaturity.get(ownerHash)?.has(snapshot.range.idString)
|
|
23256
|
+
) {
|
|
23257
|
+
continue;
|
|
23258
|
+
}
|
|
23259
|
+
try {
|
|
23260
|
+
this.schedulePendingMaturity(
|
|
23261
|
+
snapshot.pending.range,
|
|
23262
|
+
snapshot.pending.from,
|
|
23263
|
+
{
|
|
23264
|
+
rebalance: snapshot.pending.rebalance,
|
|
23265
|
+
waitMs: Math.max(0, snapshot.pending.expiresAt - Date.now()),
|
|
23266
|
+
},
|
|
23267
|
+
snapshot.pending.ownershipLifecycleController,
|
|
23268
|
+
);
|
|
23269
|
+
} catch (error) {
|
|
23270
|
+
rollbackErrors.push(error);
|
|
23271
|
+
}
|
|
23272
|
+
}
|
|
23273
|
+
try {
|
|
23274
|
+
if (wasReplicator) {
|
|
23275
|
+
this.uniqueReplicators.add(ownerHash);
|
|
23276
|
+
} else {
|
|
23277
|
+
this.uniqueReplicators.delete(ownerHash);
|
|
23278
|
+
}
|
|
23279
|
+
if (joinWasEmitted) {
|
|
23280
|
+
this._replicatorJoinEmitted.add(ownerHash);
|
|
23281
|
+
} else {
|
|
23282
|
+
this._replicatorJoinEmitted.delete(ownerHash);
|
|
23283
|
+
}
|
|
23284
|
+
} catch (error) {
|
|
23285
|
+
rollbackErrors.push(error);
|
|
23286
|
+
}
|
|
23287
|
+
try {
|
|
23288
|
+
await this.updateOldestTimestampFromIndex();
|
|
23289
|
+
} catch (error) {
|
|
23290
|
+
rollbackErrors.push(error);
|
|
23291
|
+
}
|
|
23292
|
+
if (rollbackErrors.length > 0) {
|
|
23293
|
+
const failure = new AggregateError(
|
|
23294
|
+
rollbackErrors,
|
|
23295
|
+
"Replication-range deletion rollback failed",
|
|
23296
|
+
);
|
|
23297
|
+
this.poisonReplicationOwnership(failure);
|
|
23298
|
+
throw failure;
|
|
23299
|
+
}
|
|
23300
|
+
};
|
|
22666
23301
|
return {
|
|
22667
23302
|
removed,
|
|
22668
23303
|
retained,
|
|
22669
23304
|
ownerHasRanges,
|
|
23305
|
+
rollback,
|
|
22670
23306
|
error: outcomeError,
|
|
22671
23307
|
};
|
|
22672
23308
|
}
|
|
@@ -22682,7 +23318,8 @@ export class SharedLog<
|
|
|
22682
23318
|
|
|
22683
23319
|
private scheduleReplicationInfoRequests(
|
|
22684
23320
|
peer: PublicSignKey,
|
|
22685
|
-
replicationLifecycleController = this._instanceLifecycle
|
|
23321
|
+
replicationLifecycleController = this._instanceLifecycle
|
|
23322
|
+
?.membershipLifecycleController,
|
|
22686
23323
|
) {
|
|
22687
23324
|
if (
|
|
22688
23325
|
!replicationLifecycleController ||
|
|
@@ -22764,7 +23401,8 @@ export class SharedLog<
|
|
|
22764
23401
|
if (!topics.includes(this.topic)) {
|
|
22765
23402
|
return;
|
|
22766
23403
|
}
|
|
22767
|
-
const replicationLifecycleController =
|
|
23404
|
+
const replicationLifecycleController =
|
|
23405
|
+
this._instanceLifecycle?.membershipLifecycleController;
|
|
22768
23406
|
if (
|
|
22769
23407
|
!replicationLifecycleController ||
|
|
22770
23408
|
!this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
@@ -22775,10 +23413,7 @@ export class SharedLog<
|
|
|
22775
23413
|
const peerHash = publicKey.hashcode();
|
|
22776
23414
|
const expectedSubscriptionEpoch =
|
|
22777
23415
|
subscriptionEpoch ??
|
|
22778
|
-
this._peerSessions.rotate(
|
|
22779
|
-
peerHash,
|
|
22780
|
-
subscribed ? "opening" : "departing",
|
|
22781
|
-
);
|
|
23416
|
+
this._peerSessions.rotate(peerHash, subscribed ? "opening" : "departing");
|
|
22782
23417
|
const ownsSubscriptionEpoch = () =>
|
|
22783
23418
|
this._peerSessions.isCurrent(peerHash, expectedSubscriptionEpoch);
|
|
22784
23419
|
if (!ownsSubscriptionEpoch()) {
|
|
@@ -22787,6 +23422,7 @@ export class SharedLog<
|
|
|
22787
23422
|
// A destination stream is scoped to exactly one topic-subscription
|
|
22788
23423
|
// session. Abort the predecessor synchronously before either barrier can
|
|
22789
23424
|
// yield; a late queue completion must never enter the new session.
|
|
23425
|
+
this._v2Receive.clearPeer(peerHash);
|
|
22790
23426
|
this._v2Send.clearPeer(peerHash);
|
|
22791
23427
|
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
22792
23428
|
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
@@ -22923,84 +23559,51 @@ export class SharedLog<
|
|
|
22923
23559
|
this._liveness._replicatorLivenessFailures.delete(peerHash);
|
|
22924
23560
|
this._liveness.markReplicatorActivity(peerHash);
|
|
22925
23561
|
this._peerSessions.markOpen(peerHash, expectedSubscriptionEpoch);
|
|
23562
|
+
this.promoteReplicationInfoV2ReceiveCapability(
|
|
23563
|
+
publicKey,
|
|
23564
|
+
expectedSubscriptionEpoch,
|
|
23565
|
+
);
|
|
22926
23566
|
|
|
22927
|
-
// Decode
|
|
22928
|
-
//
|
|
22929
|
-
//
|
|
22930
|
-
//
|
|
22931
|
-
|
|
22932
|
-
|
|
22933
|
-
|
|
22934
|
-
|
|
22935
|
-
|
|
22936
|
-
:
|
|
22937
|
-
|
|
22938
|
-
|
|
22939
|
-
|
|
22940
|
-
capabilities: receiveCapabilities,
|
|
22941
|
-
}),
|
|
22942
|
-
{
|
|
22943
|
-
mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
|
|
22944
|
-
signal: replicationLifecycleController.signal,
|
|
22945
|
-
},
|
|
22946
|
-
)
|
|
22947
|
-
.catch((error) =>
|
|
22948
|
-
this.handleReplicationLifecycleSendError(
|
|
22949
|
-
error,
|
|
22950
|
-
replicationLifecycleController,
|
|
22951
|
-
),
|
|
22952
|
-
);
|
|
23567
|
+
// Decode, sender and authenticated apply readiness are separate bits. An
|
|
23568
|
+
// ACKed capability advert is the local half of receiver-led negotiation;
|
|
23569
|
+
// readiness is promoted only after the legacy startup path has completed.
|
|
23570
|
+
// This preserves mixed-version discovery without putting capability ACK
|
|
23571
|
+
// latency on the subscription callback's critical path.
|
|
23572
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
23573
|
+
const localCapabilityAdvertisement =
|
|
23574
|
+
this._v2Receive.advertiseLocalCapability({
|
|
23575
|
+
target: publicKey,
|
|
23576
|
+
peerSession: expectedSubscriptionEpoch,
|
|
23577
|
+
receiveEpoch,
|
|
23578
|
+
signal: replicationLifecycleController.signal,
|
|
23579
|
+
});
|
|
22953
23580
|
|
|
22954
|
-
let replicationSegments: ReplicationRangeIndexable<R>[];
|
|
22955
23581
|
try {
|
|
22956
|
-
replicationSegments
|
|
22957
|
-
|
|
22958
|
-
|
|
22959
|
-
|
|
22960
|
-
|
|
22961
|
-
|
|
22962
|
-
|
|
23582
|
+
let replicationSegments: ReplicationRangeIndexable<R>[];
|
|
23583
|
+
try {
|
|
23584
|
+
replicationSegments = await this.getMyReplicationSegments();
|
|
23585
|
+
} catch (error) {
|
|
23586
|
+
if (
|
|
23587
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
23588
|
+
isNotStartedError(error as Error)
|
|
23589
|
+
) {
|
|
23590
|
+
return;
|
|
23591
|
+
}
|
|
23592
|
+
throw error;
|
|
22963
23593
|
}
|
|
22964
|
-
throw error;
|
|
22965
|
-
}
|
|
22966
|
-
if (
|
|
22967
|
-
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
22968
|
-
!ownsSubscriptionEpoch()
|
|
22969
|
-
) {
|
|
22970
|
-
return;
|
|
22971
|
-
}
|
|
22972
|
-
if (replicationSegments.length > 0) {
|
|
22973
|
-
const segments = replicationSegments.map((x) => x.toReplicationRange());
|
|
22974
|
-
this.validatePersistedReplicationRangeSnapshot(segments);
|
|
22975
|
-
await this.rpc
|
|
22976
|
-
.send(
|
|
22977
|
-
new AllReplicatingSegmentsMessage({
|
|
22978
|
-
segments,
|
|
22979
|
-
}),
|
|
22980
|
-
{
|
|
22981
|
-
mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] }),
|
|
22982
|
-
signal: replicationLifecycleController.signal,
|
|
22983
|
-
},
|
|
22984
|
-
)
|
|
22985
|
-
.catch((error) =>
|
|
22986
|
-
this.handleReplicationLifecycleSendError(
|
|
22987
|
-
error,
|
|
22988
|
-
replicationLifecycleController,
|
|
22989
|
-
),
|
|
22990
|
-
);
|
|
22991
23594
|
if (
|
|
22992
23595
|
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
22993
23596
|
!ownsSubscriptionEpoch()
|
|
22994
23597
|
) {
|
|
22995
23598
|
return;
|
|
22996
23599
|
}
|
|
22997
|
-
|
|
22998
|
-
|
|
22999
|
-
|
|
23600
|
+
if (replicationSegments.length > 0) {
|
|
23601
|
+
const segments = replicationSegments.map((x) => x.toReplicationRange());
|
|
23602
|
+
this.validatePersistedReplicationRangeSnapshot(segments);
|
|
23000
23603
|
await this.rpc
|
|
23001
23604
|
.send(
|
|
23002
|
-
new
|
|
23003
|
-
|
|
23605
|
+
new AllReplicatingSegmentsMessage({
|
|
23606
|
+
segments,
|
|
23004
23607
|
}),
|
|
23005
23608
|
{
|
|
23006
23609
|
mode: new AcknowledgeDelivery({
|
|
@@ -23016,20 +23619,51 @@ export class SharedLog<
|
|
|
23016
23619
|
replicationLifecycleController,
|
|
23017
23620
|
),
|
|
23018
23621
|
);
|
|
23622
|
+
if (
|
|
23623
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
23624
|
+
!ownsSubscriptionEpoch()
|
|
23625
|
+
) {
|
|
23626
|
+
return;
|
|
23627
|
+
}
|
|
23628
|
+
|
|
23629
|
+
if (this.v8Behaviour) {
|
|
23630
|
+
// for backwards compatibility
|
|
23631
|
+
await this.rpc
|
|
23632
|
+
.send(
|
|
23633
|
+
new ResponseRoleMessage({
|
|
23634
|
+
role: this.getRoleFromReplicationSegments(replicationSegments),
|
|
23635
|
+
}),
|
|
23636
|
+
{
|
|
23637
|
+
mode: new AcknowledgeDelivery({
|
|
23638
|
+
redundancy: 1,
|
|
23639
|
+
to: [publicKey],
|
|
23640
|
+
}),
|
|
23641
|
+
signal: replicationLifecycleController.signal,
|
|
23642
|
+
},
|
|
23643
|
+
)
|
|
23644
|
+
.catch((error) =>
|
|
23645
|
+
this.handleReplicationLifecycleSendError(
|
|
23646
|
+
error,
|
|
23647
|
+
replicationLifecycleController,
|
|
23648
|
+
),
|
|
23649
|
+
);
|
|
23650
|
+
}
|
|
23019
23651
|
}
|
|
23020
|
-
}
|
|
23021
23652
|
|
|
23022
|
-
|
|
23023
|
-
|
|
23024
|
-
|
|
23025
|
-
|
|
23026
|
-
|
|
23027
|
-
|
|
23028
|
-
|
|
23029
|
-
|
|
23030
|
-
|
|
23031
|
-
|
|
23032
|
-
|
|
23653
|
+
// Keep legacy request-based discovery independent of the capability ACK.
|
|
23654
|
+
// This makes mixed-version joins resilient to timing-sensitive delivery/order
|
|
23655
|
+
// issues where we may miss the remote peer's initial announcement.
|
|
23656
|
+
if (
|
|
23657
|
+
this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
23658
|
+
ownsSubscriptionEpoch()
|
|
23659
|
+
) {
|
|
23660
|
+
this.scheduleReplicationInfoRequests(
|
|
23661
|
+
publicKey,
|
|
23662
|
+
replicationLifecycleController,
|
|
23663
|
+
);
|
|
23664
|
+
}
|
|
23665
|
+
} finally {
|
|
23666
|
+
localCapabilityAdvertisement.releaseLegacyBarrier();
|
|
23033
23667
|
}
|
|
23034
23668
|
}
|
|
23035
23669
|
|
|
@@ -24153,7 +24787,10 @@ export class SharedLog<
|
|
|
24153
24787
|
if (oldPeersSet) {
|
|
24154
24788
|
for (const oldPeer of oldPeersSet) {
|
|
24155
24789
|
if (!currentPeers.has(oldPeer)) {
|
|
24156
|
-
this._checkedPrune.removeRequestSent(
|
|
24790
|
+
this._checkedPrune.removeRequestSent(
|
|
24791
|
+
entryReplicated.hash,
|
|
24792
|
+
oldPeer,
|
|
24793
|
+
);
|
|
24157
24794
|
}
|
|
24158
24795
|
}
|
|
24159
24796
|
}
|
|
@@ -24242,7 +24879,10 @@ export class SharedLog<
|
|
|
24242
24879
|
if (oldPeersSet) {
|
|
24243
24880
|
for (const oldPeer of oldPeersSet) {
|
|
24244
24881
|
if (!currentPeers.has(oldPeer)) {
|
|
24245
|
-
this._checkedPrune.removeRequestSent(
|
|
24882
|
+
this._checkedPrune.removeRequestSent(
|
|
24883
|
+
entryReplicated.hash,
|
|
24884
|
+
oldPeer,
|
|
24885
|
+
);
|
|
24246
24886
|
}
|
|
24247
24887
|
}
|
|
24248
24888
|
}
|
|
@@ -24422,10 +25062,7 @@ export class SharedLog<
|
|
|
24422
25062
|
}
|
|
24423
25063
|
|
|
24424
25064
|
const fromHash = evt.detail.from.hashcode();
|
|
24425
|
-
const subscriptionEpoch = this._peerSessions.rotate(
|
|
24426
|
-
fromHash,
|
|
24427
|
-
"departing",
|
|
24428
|
-
);
|
|
25065
|
+
const subscriptionEpoch = this._peerSessions.rotate(fromHash, "departing");
|
|
24429
25066
|
this._peerSessions.blockReplicationInfo(fromHash);
|
|
24430
25067
|
this._recentRepairDispatch.delete(fromHash);
|
|
24431
25068
|
|