@peerbit/shared-log 13.2.32 → 13.2.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +6 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +574 -92
- package/dist/src/index.js.map +1 -1
- package/dist/src/replication-info-v2-binding.d.ts +16 -0
- package/dist/src/replication-info-v2-binding.d.ts.map +1 -0
- package/dist/src/replication-info-v2-binding.js +30 -0
- package/dist/src/replication-info-v2-binding.js.map +1 -0
- package/dist/src/replication-info-v2-receive.d.ts +181 -0
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -0
- package/dist/src/replication-info-v2-receive.js +725 -0
- package/dist/src/replication-info-v2-receive.js.map +1 -0
- package/dist/src/replication-info-v2-send.d.ts +5 -17
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +15 -32
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/package.json +9 -9
- package/src/index.ts +838 -208
- package/src/replication-info-v2-binding.ts +44 -0
- package/src/replication-info-v2-receive.ts +1045 -0
- package/src/replication-info-v2-send.ts +17 -45
package/src/index.ts
CHANGED
|
@@ -130,6 +130,14 @@ import {
|
|
|
130
130
|
type CheckedPruneRetryIdentity,
|
|
131
131
|
type CheckedPruneRetryState,
|
|
132
132
|
} from "./checked-prune.js";
|
|
133
|
+
import {
|
|
134
|
+
CoordinatePersistenceCoordinator,
|
|
135
|
+
type MaybePromise,
|
|
136
|
+
combineCoordinateDeleteHashes,
|
|
137
|
+
isPromiseLike,
|
|
138
|
+
mapMaybePromise,
|
|
139
|
+
normalizedHashValues,
|
|
140
|
+
} from "./coordinate-persistence.js";
|
|
133
141
|
import { type CPUUsage, CPUUsageIntervalLag } from "./cpu.js";
|
|
134
142
|
import {
|
|
135
143
|
type DebouncedAccumulatorMap,
|
|
@@ -181,14 +189,6 @@ import {
|
|
|
181
189
|
materializeVerifiedRawExchangeHeadsMessage,
|
|
182
190
|
} from "./exchange-heads.js";
|
|
183
191
|
import { FanoutEnvelope } from "./fanout-envelope.js";
|
|
184
|
-
import {
|
|
185
|
-
CoordinatePersistenceCoordinator,
|
|
186
|
-
type MaybePromise,
|
|
187
|
-
combineCoordinateDeleteHashes,
|
|
188
|
-
isPromiseLike,
|
|
189
|
-
mapMaybePromise,
|
|
190
|
-
normalizedHashValues,
|
|
191
|
-
} from "./coordinate-persistence.js";
|
|
192
192
|
import { InstanceLifecycle } from "./instance-lifecycle.js";
|
|
193
193
|
import {
|
|
194
194
|
MAX_U32,
|
|
@@ -201,10 +201,7 @@ import { JoinWarmupCoordinator, type WarmupSession } from "./join-warmup.js";
|
|
|
201
201
|
import { LeaderPlanCache } from "./leader-plan-cache.js";
|
|
202
202
|
import { TransportMessage } from "./message.js";
|
|
203
203
|
import { NativeBackboneWriteThroughBlockStore } from "./native-write-through-block-store.js";
|
|
204
|
-
import {
|
|
205
|
-
type PeerSession,
|
|
206
|
-
PeerSessionRegistry,
|
|
207
|
-
} from "./peer-session.js";
|
|
204
|
+
import { type PeerSession, PeerSessionRegistry } from "./peer-session.js";
|
|
208
205
|
import { PIDReplicationController } from "./pid.js";
|
|
209
206
|
import {
|
|
210
207
|
type EntryReplicated,
|
|
@@ -237,7 +234,6 @@ import {
|
|
|
237
234
|
ReplicationAnnouncementCoordinator,
|
|
238
235
|
isTransientReplicationAnnouncementError,
|
|
239
236
|
} from "./replication-announcement.js";
|
|
240
|
-
import { ReplicationInfoV2SendCoordinator } from "./replication-info-v2-send.js";
|
|
241
237
|
import {
|
|
242
238
|
type ReplicationDomainHash,
|
|
243
239
|
createReplicationDomainHash,
|
|
@@ -252,18 +248,24 @@ import {
|
|
|
252
248
|
type ReplicationDomain,
|
|
253
249
|
type ReplicationDomainConstructor,
|
|
254
250
|
} from "./replication-domain.js";
|
|
251
|
+
import { ReplicationInfoV2ReceiveCoordinator } from "./replication-info-v2-receive.js";
|
|
252
|
+
import { ReplicationInfoV2SendCoordinator } from "./replication-info-v2-send.js";
|
|
255
253
|
import {
|
|
256
254
|
AbsoluteReplicas,
|
|
255
|
+
AddedReplicationInfoV2Message,
|
|
257
256
|
AddedReplicationSegmentMessage,
|
|
258
257
|
AllReplicatingSegmentsMessage,
|
|
258
|
+
FullReplicationInfoV2Message,
|
|
259
259
|
MinReplicas,
|
|
260
260
|
ReplicationError,
|
|
261
|
+
type ReplicationInfoV2Message,
|
|
261
262
|
type ReplicationLimits,
|
|
262
263
|
ReplicationPingMessage,
|
|
263
264
|
RequestReplicationInfoMessage,
|
|
264
265
|
RequestReplicationInfoV2Message,
|
|
265
266
|
ResponseRoleMessage,
|
|
266
267
|
StoppedReplicating,
|
|
268
|
+
StoppedReplicationInfoV2Message,
|
|
267
269
|
decodeReplicas,
|
|
268
270
|
encodeReplicas,
|
|
269
271
|
isReplicationInfoV2Message,
|
|
@@ -354,6 +356,7 @@ type ReceiveLaneContext = {
|
|
|
354
356
|
fromHash: string;
|
|
355
357
|
session: PeerSession | null;
|
|
356
358
|
lifecycleController: AbortController | undefined;
|
|
359
|
+
ownershipLifecycleController: AbortController;
|
|
357
360
|
receiveEpoch: object | null;
|
|
358
361
|
syncProfile: SyncProfileFn | undefined;
|
|
359
362
|
lease: PeerReceiveLease;
|
|
@@ -560,6 +563,7 @@ type ReplicationRangeDeletionOutcome<R extends "u32" | "u64"> = {
|
|
|
560
563
|
removed: ReplicationRangeIndexable<R>[];
|
|
561
564
|
retained: ReplicationRangeIndexable<R>[];
|
|
562
565
|
ownerHasRanges: boolean;
|
|
566
|
+
rollback: () => Promise<void>;
|
|
563
567
|
error?: unknown;
|
|
564
568
|
};
|
|
565
569
|
|
|
@@ -632,7 +636,8 @@ export type NativeBackboneCoordinateRollback<R extends "u32" | "u64"> = {
|
|
|
632
636
|
generations: Map<string, number>;
|
|
633
637
|
};
|
|
634
638
|
|
|
635
|
-
export type RepairDispatchEntry<R extends "u32" | "u64"> =
|
|
639
|
+
export type RepairDispatchEntry<R extends "u32" | "u64"> =
|
|
640
|
+
ResidentCoordinateEntry<R>;
|
|
636
641
|
|
|
637
642
|
type PreparedLocalAppendCommit<R extends "u32" | "u64"> = {
|
|
638
643
|
hash: string;
|
|
@@ -1091,7 +1096,6 @@ const isReplicationOptionsDependentOnPreviousState = async (
|
|
|
1091
1096
|
return false;
|
|
1092
1097
|
};
|
|
1093
1098
|
|
|
1094
|
-
|
|
1095
1099
|
export interface IndexableDomain<R extends "u32" | "u64"> {
|
|
1096
1100
|
numbers: Numbers<R>;
|
|
1097
1101
|
constructorEntry: new (properties: {
|
|
@@ -1990,7 +1994,8 @@ export class SharedLog<
|
|
|
1990
1994
|
}
|
|
1991
1995
|
|
|
1992
1996
|
get _nativeBackboneCoordinateJournalLastFlushMs(): number {
|
|
1993
|
-
return this._coordinates
|
|
1997
|
+
return this._coordinates
|
|
1998
|
+
?._nativeBackboneCoordinateJournalLastFlushMs as number;
|
|
1994
1999
|
}
|
|
1995
2000
|
|
|
1996
2001
|
set _nativeBackboneCoordinateJournalLastFlushMs(value: number) {
|
|
@@ -3077,7 +3082,8 @@ export class SharedLog<
|
|
|
3077
3082
|
if (!lowerMarkerCommitted) {
|
|
3078
3083
|
if (intent.coordinates.length > 0) {
|
|
3079
3084
|
const mutationGenerations =
|
|
3080
|
-
(this._coordinates._nativeCoordinateMutationGenerations ??=
|
|
3085
|
+
(this._coordinates._nativeCoordinateMutationGenerations ??=
|
|
3086
|
+
new Map());
|
|
3081
3087
|
const rollback: NativeBackboneCoordinateRollback<R> = {
|
|
3082
3088
|
hashes: new Set(),
|
|
3083
3089
|
entries: new Map(),
|
|
@@ -3109,7 +3115,10 @@ export class SharedLog<
|
|
|
3109
3115
|
);
|
|
3110
3116
|
}
|
|
3111
3117
|
}
|
|
3112
|
-
await this._coordinates.rollbackNativeBackboneCoordinateAppendDurably(
|
|
3118
|
+
await this._coordinates.rollbackNativeBackboneCoordinateAppendDurably(
|
|
3119
|
+
"",
|
|
3120
|
+
rollback,
|
|
3121
|
+
);
|
|
3113
3122
|
}
|
|
3114
3123
|
for (const document of intent.documents) {
|
|
3115
3124
|
this.restoreNativeBackboneDocument({
|
|
@@ -3234,6 +3243,7 @@ export class SharedLog<
|
|
|
3234
3243
|
| ReturnType<typeof debounceFixedInterval>
|
|
3235
3244
|
| undefined;
|
|
3236
3245
|
private _announcements!: ReplicationAnnouncementCoordinator<R>;
|
|
3246
|
+
private _v2Receive!: ReplicationInfoV2ReceiveCoordinator;
|
|
3237
3247
|
private _v2Send!: ReplicationInfoV2SendCoordinator<R>;
|
|
3238
3248
|
|
|
3239
3249
|
// A fn for debouncing the calls for pruning
|
|
@@ -3368,16 +3378,20 @@ export class SharedLog<
|
|
|
3368
3378
|
queueCurrentReplicationStateAnnouncementRepair: () =>
|
|
3369
3379
|
this._announcements.queueCurrentReplicationStateAnnouncementRepair(),
|
|
3370
3380
|
queueCurrentReplicationStateAnnouncementRetry: (error: unknown) =>
|
|
3371
|
-
this._announcements.queueCurrentReplicationStateAnnouncementRetry(
|
|
3381
|
+
this._announcements.queueCurrentReplicationStateAnnouncementRetry(
|
|
3382
|
+
error,
|
|
3383
|
+
),
|
|
3372
3384
|
enqueueReplicationInfoV2: (message) => this._v2Send.enqueue(message),
|
|
3373
3385
|
isClosed: () => this.closed,
|
|
3374
3386
|
getCloseSignal: () => this._closeController.signal,
|
|
3375
3387
|
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
3376
3388
|
validatePersistedReplicationRangeSnapshot: (ranges) =>
|
|
3377
3389
|
this.validatePersistedReplicationRangeSnapshot(ranges),
|
|
3378
|
-
getSubscribers: () =>
|
|
3390
|
+
getSubscribers: () =>
|
|
3391
|
+
this.node.services.pubsub.getSubscribers(this.topic),
|
|
3379
3392
|
getSelfHash: () => this.node.identity.publicKey.hashcode(),
|
|
3380
|
-
isBlockedPeer: (hash) =>
|
|
3393
|
+
isBlockedPeer: (hash) =>
|
|
3394
|
+
this._peerSessions.isReplicationInfoBlocked(hash),
|
|
3381
3395
|
getRpc: () => this.rpc,
|
|
3382
3396
|
captureReplicationOwnershipLifecycle: () =>
|
|
3383
3397
|
this.captureReplicationOwnershipLifecycle(),
|
|
@@ -3413,6 +3427,114 @@ export class SharedLog<
|
|
|
3413
3427
|
});
|
|
3414
3428
|
}
|
|
3415
3429
|
|
|
3430
|
+
private replicationInfoV2ReceiveCapabilities(): number {
|
|
3431
|
+
return (
|
|
3432
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
3433
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_SEND |
|
|
3434
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY |
|
|
3435
|
+
(this._logProperties?.sync?.rawExchangeHeads === true
|
|
3436
|
+
? SYNC_CAPABILITY_RAW_EXCHANGE_HEADS
|
|
3437
|
+
: 0)
|
|
3438
|
+
);
|
|
3439
|
+
}
|
|
3440
|
+
|
|
3441
|
+
private async advertiseReplicationInfoV2ReceiveCapability(properties: {
|
|
3442
|
+
target: PublicSignKey;
|
|
3443
|
+
peerSession: PeerSession;
|
|
3444
|
+
receiveEpoch: object | null;
|
|
3445
|
+
signal: AbortSignal;
|
|
3446
|
+
}): Promise<
|
|
3447
|
+
{ receiverTransportSession: bigint; requestNotBeforeMs: number } | undefined
|
|
3448
|
+
> {
|
|
3449
|
+
const peerHash = properties.target.hashcode();
|
|
3450
|
+
const receiverTransportSession = BigInt(
|
|
3451
|
+
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3452
|
+
);
|
|
3453
|
+
await this.rpc.send(
|
|
3454
|
+
new SyncCapabilitiesMessage({
|
|
3455
|
+
capabilities: this.replicationInfoV2ReceiveCapabilities(),
|
|
3456
|
+
}),
|
|
3457
|
+
{
|
|
3458
|
+
mode: new AcknowledgeDelivery({
|
|
3459
|
+
redundancy: 1,
|
|
3460
|
+
to: [properties.target],
|
|
3461
|
+
}),
|
|
3462
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
3463
|
+
signal: properties.signal,
|
|
3464
|
+
},
|
|
3465
|
+
);
|
|
3466
|
+
if (
|
|
3467
|
+
properties.signal.aborted ||
|
|
3468
|
+
this.closed ||
|
|
3469
|
+
!this._peerSessions.isCurrent(peerHash, properties.peerSession) ||
|
|
3470
|
+
properties.peerSession.phase !== "open" ||
|
|
3471
|
+
!properties.peerSession.isActive() ||
|
|
3472
|
+
!this._peerSessions.isReceiveEpochCurrent(
|
|
3473
|
+
peerHash,
|
|
3474
|
+
properties.receiveEpoch,
|
|
3475
|
+
) ||
|
|
3476
|
+
this._peerSessions.isReplicationInfoBlocked(peerHash) ||
|
|
3477
|
+
!this._peerSessions.isReceiveCleanupGateOpen(peerHash) ||
|
|
3478
|
+
BigInt(
|
|
3479
|
+
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3480
|
+
) !== receiverTransportSession
|
|
3481
|
+
) {
|
|
3482
|
+
return undefined;
|
|
3483
|
+
}
|
|
3484
|
+
return {
|
|
3485
|
+
receiverTransportSession,
|
|
3486
|
+
requestNotBeforeMs: Date.now(),
|
|
3487
|
+
};
|
|
3488
|
+
}
|
|
3489
|
+
|
|
3490
|
+
private createReplicationInfoV2ReceiveCoordinator(): ReplicationInfoV2ReceiveCoordinator {
|
|
3491
|
+
return new ReplicationInfoV2ReceiveCoordinator({
|
|
3492
|
+
getSelfKey: () => this.node.identity.publicKey,
|
|
3493
|
+
getReceiverTransportSession: () =>
|
|
3494
|
+
BigInt(
|
|
3495
|
+
(this.node.services.pubsub as unknown as { session: number }).session,
|
|
3496
|
+
),
|
|
3497
|
+
isClosed: () => this.closed,
|
|
3498
|
+
isPeerStateCurrent: (peerHash, peerSession, receiveEpoch) =>
|
|
3499
|
+
this._peerSessions.isCurrent(peerHash, peerSession) &&
|
|
3500
|
+
(peerSession as PeerSession).phase === "open" &&
|
|
3501
|
+
(peerSession as PeerSession).isActive() &&
|
|
3502
|
+
this._peerSessions.isReceiveEpochCurrent(peerHash, receiveEpoch) &&
|
|
3503
|
+
!this._peerSessions.isReplicationInfoBlocked(peerHash) &&
|
|
3504
|
+
this._peerSessions.isReceiveCleanupGateOpen(peerHash),
|
|
3505
|
+
isSenderTransportSessionCurrent: (peerHash, senderTransportSession) =>
|
|
3506
|
+
this._peerSyncCapabilitySessions.get(peerHash) ===
|
|
3507
|
+
senderTransportSession,
|
|
3508
|
+
sendRequest: async (request, target, signal) => {
|
|
3509
|
+
await this.rpc.send(request, {
|
|
3510
|
+
mode: new AcknowledgeDelivery({ redundancy: 1, to: [target] }),
|
|
3511
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
3512
|
+
signal,
|
|
3513
|
+
});
|
|
3514
|
+
},
|
|
3515
|
+
refreshLocalCapability: (properties) =>
|
|
3516
|
+
this.advertiseReplicationInfoV2ReceiveCapability({
|
|
3517
|
+
target: properties.target,
|
|
3518
|
+
peerSession: properties.peerSession as PeerSession,
|
|
3519
|
+
receiveEpoch: properties.receiveEpoch,
|
|
3520
|
+
signal: properties.signal,
|
|
3521
|
+
}),
|
|
3522
|
+
onRequestError: (error) => {
|
|
3523
|
+
if (
|
|
3524
|
+
isNotStartedError(error as Error) ||
|
|
3525
|
+
(this.closed && error instanceof AbortError)
|
|
3526
|
+
) {
|
|
3527
|
+
return;
|
|
3528
|
+
}
|
|
3529
|
+
logger.trace(
|
|
3530
|
+
`Replication-info V2 recovery request failed: ${
|
|
3531
|
+
(error as Error)?.message ?? String(error)
|
|
3532
|
+
}`,
|
|
3533
|
+
);
|
|
3534
|
+
},
|
|
3535
|
+
});
|
|
3536
|
+
}
|
|
3537
|
+
|
|
3416
3538
|
private createReplicatorLivenessMonitor(): ReplicatorLivenessMonitor {
|
|
3417
3539
|
return new ReplicatorLivenessMonitor({
|
|
3418
3540
|
// Sweep-driven probes and activity marks dispatch through the owner so
|
|
@@ -3443,7 +3565,8 @@ export class SharedLog<
|
|
|
3443
3565
|
}),
|
|
3444
3566
|
);
|
|
3445
3567
|
},
|
|
3446
|
-
isBlockedPeer: (hash) =>
|
|
3568
|
+
isBlockedPeer: (hash) =>
|
|
3569
|
+
this._peerSessions.isReplicationInfoBlocked(hash),
|
|
3447
3570
|
scheduleReplicationInfoRequests: (peer, replicationLifecycleController) =>
|
|
3448
3571
|
this.scheduleReplicationInfoRequests(
|
|
3449
3572
|
peer,
|
|
@@ -3537,6 +3660,7 @@ export class SharedLog<
|
|
|
3537
3660
|
this._replicatorJoinEmitted = new Set();
|
|
3538
3661
|
this._replicatorsReconciled = false;
|
|
3539
3662
|
this._liveness = this.createReplicatorLivenessMonitor();
|
|
3663
|
+
this._v2Receive = this.createReplicationInfoV2ReceiveCoordinator();
|
|
3540
3664
|
this._v2Send = this.createReplicationInfoV2SendCoordinator();
|
|
3541
3665
|
this._announcements = this.createReplicationAnnouncementCoordinator();
|
|
3542
3666
|
this.pendingMaturity = new Map();
|
|
@@ -4049,13 +4173,26 @@ export class SharedLog<
|
|
|
4049
4173
|
} = properties;
|
|
4050
4174
|
if (openingSession) {
|
|
4051
4175
|
const previous = this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
4176
|
+
if (
|
|
4177
|
+
transportSession !== undefined &&
|
|
4178
|
+
timestamp !== undefined &&
|
|
4179
|
+
previous?.epoch === openingSession &&
|
|
4180
|
+
previous.timestamp !== undefined &&
|
|
4181
|
+
previous.transportSession !== transportSession &&
|
|
4182
|
+
timestamp <= previous.timestamp
|
|
4183
|
+
) {
|
|
4184
|
+
return false;
|
|
4185
|
+
}
|
|
4052
4186
|
if (
|
|
4053
4187
|
transportSession !== undefined &&
|
|
4054
4188
|
timestamp !== undefined &&
|
|
4055
4189
|
previous?.epoch === openingSession &&
|
|
4056
4190
|
previous.transportSession === transportSession
|
|
4057
4191
|
) {
|
|
4058
|
-
if (
|
|
4192
|
+
if (
|
|
4193
|
+
previous.timestamp !== undefined &&
|
|
4194
|
+
timestamp < previous.timestamp
|
|
4195
|
+
) {
|
|
4059
4196
|
return false;
|
|
4060
4197
|
}
|
|
4061
4198
|
this._openingSyncCapabilitiesByPeer.set(peerHash, {
|
|
@@ -4085,6 +4222,7 @@ export class SharedLog<
|
|
|
4085
4222
|
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
4086
4223
|
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
4087
4224
|
this._v2Send.advancePeerCapability(peerHash);
|
|
4225
|
+
this._v2Receive.revokePeerCapability(peerHash);
|
|
4088
4226
|
return true;
|
|
4089
4227
|
}
|
|
4090
4228
|
|
|
@@ -4092,22 +4230,23 @@ export class SharedLog<
|
|
|
4092
4230
|
const previousTimestamp = this._peerSyncCapabilityTimestamps.get(peerHash);
|
|
4093
4231
|
const sameTransportSession = previousSession === transportSession;
|
|
4094
4232
|
if (
|
|
4095
|
-
sameTransportSession &&
|
|
4096
4233
|
previousTimestamp !== undefined &&
|
|
4097
|
-
timestamp < previousTimestamp
|
|
4234
|
+
((sameTransportSession && timestamp < previousTimestamp) ||
|
|
4235
|
+
(!sameTransportSession && timestamp <= previousTimestamp))
|
|
4098
4236
|
) {
|
|
4099
4237
|
return false;
|
|
4100
4238
|
}
|
|
4101
|
-
const previousCapabilities =
|
|
4102
|
-
this._peerSyncCapabilities.get(peerHash) ?? 0;
|
|
4239
|
+
const previousCapabilities = this._peerSyncCapabilities.get(peerHash) ?? 0;
|
|
4103
4240
|
const nextCapabilities = sameTransportSession
|
|
4104
4241
|
? previousCapabilities | capabilities
|
|
4105
4242
|
: capabilities;
|
|
4243
|
+
const senderGrantCapabilityMask =
|
|
4244
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_DECODE |
|
|
4245
|
+
SYNC_CAPABILITY_REPLICATION_INFO_V2_APPLY;
|
|
4106
4246
|
const generationAdvanced =
|
|
4107
4247
|
!sameTransportSession ||
|
|
4108
|
-
|
|
4109
|
-
|
|
4110
|
-
nextCapabilities !== previousCapabilities;
|
|
4248
|
+
(previousCapabilities & senderGrantCapabilityMask) !==
|
|
4249
|
+
(nextCapabilities & senderGrantCapabilityMask);
|
|
4111
4250
|
this._peerSyncCapabilities.set(peerHash, nextCapabilities);
|
|
4112
4251
|
this._peerSyncCapabilitySessions.set(peerHash, transportSession);
|
|
4113
4252
|
this._peerSyncCapabilityTimestamps.set(
|
|
@@ -4124,6 +4263,32 @@ export class SharedLog<
|
|
|
4124
4263
|
return true;
|
|
4125
4264
|
}
|
|
4126
4265
|
|
|
4266
|
+
private promoteReplicationInfoV2ReceiveCapability(
|
|
4267
|
+
target: PublicSignKey,
|
|
4268
|
+
peerSession: PeerSession,
|
|
4269
|
+
): boolean {
|
|
4270
|
+
const peerHash = target.hashcode();
|
|
4271
|
+
const senderTransportSession =
|
|
4272
|
+
this._peerSyncCapabilitySessions.get(peerHash);
|
|
4273
|
+
const capabilityTimestamp =
|
|
4274
|
+
this._peerSyncCapabilityTimestamps.get(peerHash);
|
|
4275
|
+
if (
|
|
4276
|
+
senderTransportSession === undefined ||
|
|
4277
|
+
capabilityTimestamp === undefined
|
|
4278
|
+
) {
|
|
4279
|
+
return false;
|
|
4280
|
+
}
|
|
4281
|
+
return this._v2Receive.observeCapability({
|
|
4282
|
+
peerHash,
|
|
4283
|
+
target,
|
|
4284
|
+
peerSession,
|
|
4285
|
+
receiveEpoch: this._peerSessions.receiveEpoch(peerHash),
|
|
4286
|
+
capabilities: this._peerSyncCapabilities.get(peerHash) ?? 0,
|
|
4287
|
+
senderTransportSession,
|
|
4288
|
+
capabilityTimestamp,
|
|
4289
|
+
});
|
|
4290
|
+
}
|
|
4291
|
+
|
|
4127
4292
|
/**
|
|
4128
4293
|
* Live append gossip may use the raw exchange-heads path only when we
|
|
4129
4294
|
* opted into raw sync and every remote recipient advertised raw capability
|
|
@@ -5001,7 +5166,8 @@ export class SharedLog<
|
|
|
5001
5166
|
|
|
5002
5167
|
private isReceiveOwnershipSnapshotStable(revision: number): boolean {
|
|
5003
5168
|
return (
|
|
5004
|
-
(this._instanceLifecycle?._receiveOwnershipMutationAdmissions ?? 0) ===
|
|
5169
|
+
(this._instanceLifecycle?._receiveOwnershipMutationAdmissions ?? 0) ===
|
|
5170
|
+
0 &&
|
|
5005
5171
|
(this._instanceLifecycle?._receiveOwnershipRevision ?? 0) === revision
|
|
5006
5172
|
);
|
|
5007
5173
|
}
|
|
@@ -5249,7 +5415,8 @@ export class SharedLog<
|
|
|
5249
5415
|
pending: PendingIHave<T>,
|
|
5250
5416
|
entry: Entry<T>,
|
|
5251
5417
|
): void {
|
|
5252
|
-
const replicationLifecycleController =
|
|
5418
|
+
const replicationLifecycleController =
|
|
5419
|
+
this._instanceLifecycle?.membershipLifecycleController;
|
|
5253
5420
|
if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
5254
5421
|
if (this._pendingIHave.get(entry.hash) === pending) {
|
|
5255
5422
|
pending.clear();
|
|
@@ -6157,6 +6324,7 @@ export class SharedLog<
|
|
|
6157
6324
|
};
|
|
6158
6325
|
let removed = false;
|
|
6159
6326
|
let removalCallCompleted = false;
|
|
6327
|
+
let replicationInfoRecoveryEpochAdvanced = false;
|
|
6160
6328
|
|
|
6161
6329
|
// Replication-info updates already serialize per peer. Put the hash-wide
|
|
6162
6330
|
// removal on the same queue so a newer reset cannot be deleted underneath
|
|
@@ -6246,6 +6414,13 @@ export class SharedLog<
|
|
|
6246
6414
|
checkedPruneCoordinator,
|
|
6247
6415
|
);
|
|
6248
6416
|
}
|
|
6417
|
+
if (!isMe) {
|
|
6418
|
+
// This is the last synchronous current-generation boundary
|
|
6419
|
+
// before destructive work. Fence parked receives now because
|
|
6420
|
+
// the coherent delete can durably mutate and then fail or poison.
|
|
6421
|
+
this.advanceReplicationInfoRecoveryEpoch(keyHash);
|
|
6422
|
+
replicationInfoRecoveryEpochAdvanced = true;
|
|
6423
|
+
}
|
|
6249
6424
|
const deletion = await this.deleteReplicationRangesCoherently(
|
|
6250
6425
|
deleted,
|
|
6251
6426
|
keyHash,
|
|
@@ -6269,7 +6444,6 @@ export class SharedLog<
|
|
|
6269
6444
|
}
|
|
6270
6445
|
return;
|
|
6271
6446
|
}
|
|
6272
|
-
|
|
6273
6447
|
if (options?.noEvent !== true && deleted.length > 0) {
|
|
6274
6448
|
const publicKey = toLocalPublicSignKey(key);
|
|
6275
6449
|
if (publicKey) {
|
|
@@ -6305,11 +6479,6 @@ export class SharedLog<
|
|
|
6305
6479
|
await cleanupDisconnectedPeer();
|
|
6306
6480
|
|
|
6307
6481
|
if (!isMe) {
|
|
6308
|
-
// Replication-info handlers release their receive lease before joining
|
|
6309
|
-
// this lane. Fence every handler queued behind this successful removal,
|
|
6310
|
-
// regardless of whether it came from liveness, startup pruning, or an
|
|
6311
|
-
// unsubscribe transition.
|
|
6312
|
-
this.advanceReplicationInfoRecoveryEpoch(keyHash);
|
|
6313
6482
|
this.rebalanceParticipationDebounced?.call();
|
|
6314
6483
|
}
|
|
6315
6484
|
removed = true;
|
|
@@ -6347,6 +6516,24 @@ export class SharedLog<
|
|
|
6347
6516
|
removalCallCompleted = true;
|
|
6348
6517
|
} finally {
|
|
6349
6518
|
releaseReceiveCleanupGate?.();
|
|
6519
|
+
if (
|
|
6520
|
+
replicationInfoRecoveryEpochAdvanced &&
|
|
6521
|
+
ownsReplicationOwnershipLifecycle() &&
|
|
6522
|
+
ownsReplicationLifecycle() &&
|
|
6523
|
+
ownsSubscriptionEpoch()
|
|
6524
|
+
) {
|
|
6525
|
+
// The first recovery arm ran while receive admission was fenced.
|
|
6526
|
+
// Re-arm after releasing the cleanup gate so a still-open peer can
|
|
6527
|
+
// deliver the authoritative Full that completes recovery.
|
|
6528
|
+
const peerSession = this._peerSessions.current(keyHash);
|
|
6529
|
+
if (peerSession?.phase === "open") {
|
|
6530
|
+
this._v2Receive.advanceRecovery({
|
|
6531
|
+
peerHash: keyHash,
|
|
6532
|
+
peerSession,
|
|
6533
|
+
receiveEpoch: this._peerSessions.receiveEpoch(keyHash),
|
|
6534
|
+
});
|
|
6535
|
+
}
|
|
6536
|
+
}
|
|
6350
6537
|
if (isSpeculativePeerRemoval) {
|
|
6351
6538
|
const cancelledByFreshActivity =
|
|
6352
6539
|
removalCallCompleted &&
|
|
@@ -6422,6 +6609,7 @@ export class SharedLog<
|
|
|
6422
6609
|
options?: {
|
|
6423
6610
|
onRemoved?: (ranges: ReplicationRangeIndexable<R>[]) => void;
|
|
6424
6611
|
shouldRemove?: () => boolean;
|
|
6612
|
+
onDurableRemoveCommitted?: () => boolean | void;
|
|
6425
6613
|
},
|
|
6426
6614
|
replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
6427
6615
|
): Promise<boolean> {
|
|
@@ -6440,6 +6628,7 @@ export class SharedLog<
|
|
|
6440
6628
|
options?: {
|
|
6441
6629
|
onRemoved?: (ranges: ReplicationRangeIndexable<R>[]) => void;
|
|
6442
6630
|
shouldRemove?: () => boolean;
|
|
6631
|
+
onDurableRemoveCommitted?: () => boolean | void;
|
|
6443
6632
|
},
|
|
6444
6633
|
): Promise<boolean> {
|
|
6445
6634
|
if (ranges.length === 0) {
|
|
@@ -6496,6 +6685,13 @@ export class SharedLog<
|
|
|
6496
6685
|
ownerHash,
|
|
6497
6686
|
);
|
|
6498
6687
|
ranges = deletion.removed;
|
|
6688
|
+
if (
|
|
6689
|
+
(options?.shouldRemove && !options.shouldRemove()) ||
|
|
6690
|
+
options?.onDurableRemoveCommitted?.() === false
|
|
6691
|
+
) {
|
|
6692
|
+
await deletion.rollback();
|
|
6693
|
+
return false;
|
|
6694
|
+
}
|
|
6499
6695
|
options?.onRemoved?.(ranges);
|
|
6500
6696
|
|
|
6501
6697
|
if (ranges.length > 0) {
|
|
@@ -6581,6 +6777,7 @@ export class SharedLog<
|
|
|
6581
6777
|
timestamp?: number;
|
|
6582
6778
|
allowLegacyOrderedReplacementPairs?: boolean;
|
|
6583
6779
|
onConfirmedDurableStateChanged?: () => void;
|
|
6780
|
+
onDurableApplyCommitted?: () => boolean | void;
|
|
6584
6781
|
shouldApply?: () => boolean;
|
|
6585
6782
|
} = {},
|
|
6586
6783
|
replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
@@ -6622,6 +6819,7 @@ export class SharedLog<
|
|
|
6622
6819
|
rebalance,
|
|
6623
6820
|
allowLegacyOrderedReplacementPairs,
|
|
6624
6821
|
onConfirmedDurableStateChanged,
|
|
6822
|
+
onDurableApplyCommitted,
|
|
6625
6823
|
shouldApply,
|
|
6626
6824
|
}: {
|
|
6627
6825
|
reset?: boolean;
|
|
@@ -6630,6 +6828,7 @@ export class SharedLog<
|
|
|
6630
6828
|
timestamp?: number;
|
|
6631
6829
|
allowLegacyOrderedReplacementPairs?: boolean;
|
|
6632
6830
|
onConfirmedDurableStateChanged?: () => void;
|
|
6831
|
+
onDurableApplyCommitted?: () => boolean | void;
|
|
6633
6832
|
shouldApply?: () => boolean;
|
|
6634
6833
|
} = {},
|
|
6635
6834
|
replicationOwnershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
@@ -6644,6 +6843,12 @@ export class SharedLog<
|
|
|
6644
6843
|
if (shouldApply && !shouldApply()) {
|
|
6645
6844
|
return [];
|
|
6646
6845
|
}
|
|
6846
|
+
const applySuperseded = Symbol("replication-range-apply-superseded");
|
|
6847
|
+
const throwIfApplySuperseded = () => {
|
|
6848
|
+
if (shouldApply && !shouldApply()) {
|
|
6849
|
+
throw applySuperseded;
|
|
6850
|
+
}
|
|
6851
|
+
};
|
|
6647
6852
|
const fromHash = from.hashcode();
|
|
6648
6853
|
const incomingRangesById = new Map<
|
|
6649
6854
|
string,
|
|
@@ -6733,6 +6938,7 @@ export class SharedLog<
|
|
|
6733
6938
|
let isStoppedReplicating = false;
|
|
6734
6939
|
let wasReplicatorBeforeDestructiveReset = false;
|
|
6735
6940
|
let resetFailureLeaveEmitted = false;
|
|
6941
|
+
let resetDeletionRollback: (() => Promise<void>) | undefined;
|
|
6736
6942
|
const publishConfirmedResetStop = (ownerHasRanges: boolean) => {
|
|
6737
6943
|
if (ownerHasRanges || resetFailureLeaveEmitted) {
|
|
6738
6944
|
return;
|
|
@@ -6785,12 +6991,18 @@ export class SharedLog<
|
|
|
6785
6991
|
} else {
|
|
6786
6992
|
wasReplicatorBeforeDestructiveReset =
|
|
6787
6993
|
this.uniqueReplicators.has(fromHash);
|
|
6994
|
+
throwIfApplySuperseded();
|
|
6788
6995
|
const deletion = await this.deleteReplicationRangesCoherently(
|
|
6789
6996
|
deleted,
|
|
6790
6997
|
fromHash,
|
|
6791
6998
|
{ preserveOwnerMembership: ranges.length > 0 },
|
|
6792
6999
|
);
|
|
6793
7000
|
deleted = deletion.removed;
|
|
7001
|
+
resetDeletionRollback = deletion.rollback;
|
|
7002
|
+
if (shouldApply && !shouldApply()) {
|
|
7003
|
+
await deletion.rollback();
|
|
7004
|
+
return [];
|
|
7005
|
+
}
|
|
6794
7006
|
|
|
6795
7007
|
diffs = [
|
|
6796
7008
|
...deleted.map((x) => {
|
|
@@ -6986,18 +7198,26 @@ export class SharedLog<
|
|
|
6986
7198
|
};
|
|
6987
7199
|
|
|
6988
7200
|
try {
|
|
7201
|
+
throwIfApplySuperseded();
|
|
6989
7202
|
for (const diff of diffs) {
|
|
6990
7203
|
if (diff.type !== "added") {
|
|
6991
7204
|
continue;
|
|
6992
7205
|
}
|
|
7206
|
+
throwIfApplySuperseded();
|
|
6993
7207
|
appliedPositiveRanges.push(diff);
|
|
6994
7208
|
await this.replicationIndex.put(diff.range);
|
|
6995
7209
|
this.putNativeReplicationRange(diff.range);
|
|
7210
|
+
throwIfApplySuperseded();
|
|
6996
7211
|
}
|
|
6997
7212
|
if (reset && diffs.length > 0) {
|
|
6998
7213
|
await this.updateOldestTimestampFromIndex();
|
|
7214
|
+
throwIfApplySuperseded();
|
|
7215
|
+
}
|
|
7216
|
+
if (onDurableApplyCommitted?.() === false) {
|
|
7217
|
+
throw applySuperseded;
|
|
6999
7218
|
}
|
|
7000
7219
|
} catch (error) {
|
|
7220
|
+
const superseded = error === applySuperseded;
|
|
7001
7221
|
let outcomeError = error;
|
|
7002
7222
|
if (appliedPositiveRanges.length > 0) {
|
|
7003
7223
|
try {
|
|
@@ -7006,6 +7226,19 @@ export class SharedLog<
|
|
|
7006
7226
|
outcomeError = poisonFromPositiveRollback(rollbackError, error);
|
|
7007
7227
|
}
|
|
7008
7228
|
}
|
|
7229
|
+
if (superseded) {
|
|
7230
|
+
if (resetDeletionRollback) {
|
|
7231
|
+
try {
|
|
7232
|
+
await resetDeletionRollback();
|
|
7233
|
+
} catch (rollbackError) {
|
|
7234
|
+
outcomeError = rollbackError;
|
|
7235
|
+
}
|
|
7236
|
+
}
|
|
7237
|
+
if (outcomeError === applySuperseded) {
|
|
7238
|
+
return [];
|
|
7239
|
+
}
|
|
7240
|
+
throw outcomeError;
|
|
7241
|
+
}
|
|
7009
7242
|
if (reset) {
|
|
7010
7243
|
const negativeDiffs = diffs.filter((diff) => diff.type !== "added");
|
|
7011
7244
|
if (negativeDiffs.length > 0) {
|
|
@@ -8612,7 +8845,8 @@ export class SharedLog<
|
|
|
8612
8845
|
}
|
|
8613
8846
|
};
|
|
8614
8847
|
|
|
8615
|
-
const residentEntriesByHash =
|
|
8848
|
+
const residentEntriesByHash =
|
|
8849
|
+
this._coordinates._residentEntryCoordinatesByHash;
|
|
8616
8850
|
if (
|
|
8617
8851
|
(this._nativeBackbone ?? this._nativeSharedLogState) &&
|
|
8618
8852
|
residentEntriesByHash &&
|
|
@@ -8924,7 +9158,8 @@ export class SharedLog<
|
|
|
8924
9158
|
const localLeaderHashes = new Set<string>();
|
|
8925
9159
|
const unresolvedHashes = new Set(hashes);
|
|
8926
9160
|
const nativePlanner = this._nativeBackbone ?? this._nativeRangePlanner;
|
|
8927
|
-
const nativeEntryMetadata =
|
|
9161
|
+
const nativeEntryMetadata =
|
|
9162
|
+
this._coordinates.getNativeLogEntryMetadataBatch(hashes);
|
|
8928
9163
|
|
|
8929
9164
|
if (nativePlanner && !this.hasCustomFindLeaders() && nativeEntryMetadata) {
|
|
8930
9165
|
const nativeItems: Array<{
|
|
@@ -10425,7 +10660,8 @@ export class SharedLog<
|
|
|
10425
10660
|
) {
|
|
10426
10661
|
try {
|
|
10427
10662
|
this.restoreNativeBackboneDocument(nativeDocumentRollback);
|
|
10428
|
-
const flushed =
|
|
10663
|
+
const flushed =
|
|
10664
|
+
this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
10429
10665
|
if (isPromiseLike(flushed)) {
|
|
10430
10666
|
await flushed;
|
|
10431
10667
|
}
|
|
@@ -10515,11 +10751,12 @@ export class SharedLog<
|
|
|
10515
10751
|
prepared.trimmedEntries as Array<{ hash?: string }> | undefined
|
|
10516
10752
|
)?.flatMap((entry) => (entry.hash ? [entry.hash] : [])) ??
|
|
10517
10753
|
[];
|
|
10518
|
-
const coordinateRollback =
|
|
10519
|
-
|
|
10520
|
-
|
|
10521
|
-
|
|
10522
|
-
|
|
10754
|
+
const coordinateRollback =
|
|
10755
|
+
this._coordinates.snapshotResidentCoordinateEntries([
|
|
10756
|
+
...(preparedHash ? [preparedHash] : []),
|
|
10757
|
+
...preparedNext,
|
|
10758
|
+
...nativeTrimmedHashes,
|
|
10759
|
+
]);
|
|
10523
10760
|
lowerPublicationRollback = {
|
|
10524
10761
|
committedHashes: preparedHash ? [preparedHash] : [],
|
|
10525
10762
|
trimmedEntries: prepared.trimmedEntries,
|
|
@@ -10735,7 +10972,8 @@ export class SharedLog<
|
|
|
10735
10972
|
for (const document of lowerPublicationRollback?.documents ?? []) {
|
|
10736
10973
|
this.restoreNativeBackboneDocument(document);
|
|
10737
10974
|
}
|
|
10738
|
-
const flushed =
|
|
10975
|
+
const flushed =
|
|
10976
|
+
this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
10739
10977
|
if (isPromiseLike(flushed)) {
|
|
10740
10978
|
await flushed;
|
|
10741
10979
|
}
|
|
@@ -10830,7 +11068,10 @@ export class SharedLog<
|
|
|
10830
11068
|
ownershipLifecycleController,
|
|
10831
11069
|
);
|
|
10832
11070
|
const backbone = this._nativeBackbone;
|
|
10833
|
-
if (
|
|
11071
|
+
if (
|
|
11072
|
+
!backbone ||
|
|
11073
|
+
!this._coordinates.canUseNativeBackboneResidentCoordinateState()
|
|
11074
|
+
) {
|
|
10834
11075
|
return undefined;
|
|
10835
11076
|
}
|
|
10836
11077
|
if (
|
|
@@ -11035,11 +11276,12 @@ export class SharedLog<
|
|
|
11035
11276
|
const committedHash =
|
|
11036
11277
|
backboneAppend.entry.cid ?? backboneAppend.entry.hash;
|
|
11037
11278
|
const committedNext = backboneAppend.entry.next ?? next;
|
|
11038
|
-
nativeCoordinateRollback =
|
|
11039
|
-
|
|
11040
|
-
|
|
11041
|
-
|
|
11042
|
-
|
|
11279
|
+
nativeCoordinateRollback =
|
|
11280
|
+
this._coordinates.snapshotResidentCoordinateEntries([
|
|
11281
|
+
...(committedHash ? [committedHash] : []),
|
|
11282
|
+
...committedNext,
|
|
11283
|
+
...nativeTrimmedHashes,
|
|
11284
|
+
]);
|
|
11043
11285
|
await this.setNativeStrictDurableTransactionOperation(
|
|
11044
11286
|
nativeStrictTransaction,
|
|
11045
11287
|
committedHash ? [committedHash] : [],
|
|
@@ -11350,7 +11592,8 @@ export class SharedLog<
|
|
|
11350
11592
|
if (nativeDocumentRollback) {
|
|
11351
11593
|
try {
|
|
11352
11594
|
this.restoreNativeBackboneDocument(nativeDocumentRollback);
|
|
11353
|
-
const flushed =
|
|
11595
|
+
const flushed =
|
|
11596
|
+
this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
11354
11597
|
if (isPromiseLike(flushed)) {
|
|
11355
11598
|
await flushed;
|
|
11356
11599
|
}
|
|
@@ -11382,15 +11625,17 @@ export class SharedLog<
|
|
|
11382
11625
|
coordinateIndex.putSharedLogCoordinateFieldsEncodedAndDeleteHashesNoReturn ||
|
|
11383
11626
|
coordinateIndex.putSharedLogCoordinateFieldsAndDeleteHashesNoReturn;
|
|
11384
11627
|
const persisted = hasNativeCoordinatePut
|
|
11385
|
-
? this._coordinates.persistBackboneCoordinateFieldsNativeTransaction(
|
|
11386
|
-
|
|
11387
|
-
|
|
11388
|
-
|
|
11389
|
-
|
|
11390
|
-
|
|
11391
|
-
|
|
11392
|
-
|
|
11393
|
-
|
|
11628
|
+
? this._coordinates.persistBackboneCoordinateFieldsNativeTransaction(
|
|
11629
|
+
{
|
|
11630
|
+
coordinateIndex,
|
|
11631
|
+
fields: coordinateFields,
|
|
11632
|
+
hash: prepared.appendFacts.hash,
|
|
11633
|
+
deleteHashes: [],
|
|
11634
|
+
coordinates: backboneAppend.coordinate
|
|
11635
|
+
.coordinates as NumberFromType<R>[],
|
|
11636
|
+
skipGenericTransientCoordinateIndex: runtimeOnlyCoordinates,
|
|
11637
|
+
},
|
|
11638
|
+
)
|
|
11394
11639
|
: this._coordinates.persistPreparedCoordinate({
|
|
11395
11640
|
prepared: getPreparedCoordinate(),
|
|
11396
11641
|
hash: prepared.appendFacts.hash,
|
|
@@ -11444,9 +11689,10 @@ export class SharedLog<
|
|
|
11444
11689
|
if (!backboneAppend.isLeader && !delayAdaptiveRebalance) {
|
|
11445
11690
|
const leaders = backboneAppend.leaders;
|
|
11446
11691
|
if (leaders) {
|
|
11447
|
-
const pruneEntry =
|
|
11448
|
-
|
|
11449
|
-
|
|
11692
|
+
const pruneEntry =
|
|
11693
|
+
this._coordinates.materializePreparedCoordinateEntry(
|
|
11694
|
+
getPreparedCoordinate(),
|
|
11695
|
+
);
|
|
11450
11696
|
this.pruneDebouncedFnAddIfNotKeeping({
|
|
11451
11697
|
key: pruneEntry.hash,
|
|
11452
11698
|
value: { entry: pruneEntry, leaders },
|
|
@@ -12117,16 +12363,17 @@ export class SharedLog<
|
|
|
12117
12363
|
),
|
|
12118
12364
|
);
|
|
12119
12365
|
const nativeTrimmedHashes = [...nativeTrimmedHashSet];
|
|
12120
|
-
batchCoordinateRollback =
|
|
12121
|
-
|
|
12122
|
-
|
|
12123
|
-
|
|
12124
|
-
|
|
12125
|
-
|
|
12126
|
-
|
|
12127
|
-
append.
|
|
12128
|
-
|
|
12129
|
-
|
|
12366
|
+
batchCoordinateRollback =
|
|
12367
|
+
this._coordinates.snapshotResidentCoordinateEntries(
|
|
12368
|
+
committedAppends.flatMap((append) => [
|
|
12369
|
+
...((append.entry.cid ?? append.entry.hash)
|
|
12370
|
+
? [append.entry.cid ?? append.entry.hash!]
|
|
12371
|
+
: []),
|
|
12372
|
+
...append.entry.next,
|
|
12373
|
+
...(append.trimmedHashes ??
|
|
12374
|
+
append.trimmed.map((entry) => entry.hash)),
|
|
12375
|
+
]),
|
|
12376
|
+
);
|
|
12130
12377
|
await this.setNativeStrictDurableTransactionOperation(
|
|
12131
12378
|
nativeStrictTransaction,
|
|
12132
12379
|
committedCids,
|
|
@@ -12378,7 +12625,8 @@ export class SharedLog<
|
|
|
12378
12625
|
for (const document of batchDocumentRollbacks) {
|
|
12379
12626
|
this.restoreNativeBackboneDocument(document);
|
|
12380
12627
|
}
|
|
12381
|
-
const flushed =
|
|
12628
|
+
const flushed =
|
|
12629
|
+
this._coordinates.flushNativeBackboneCoordinateJournal();
|
|
12382
12630
|
if (isPromiseLike(flushed)) {
|
|
12383
12631
|
await flushed;
|
|
12384
12632
|
}
|
|
@@ -12458,20 +12706,21 @@ export class SharedLog<
|
|
|
12458
12706
|
coordinateFields,
|
|
12459
12707
|
plannedCoordinateDeleteHashes,
|
|
12460
12708
|
});
|
|
12461
|
-
const persisted =
|
|
12462
|
-
|
|
12463
|
-
|
|
12464
|
-
|
|
12465
|
-
|
|
12466
|
-
|
|
12467
|
-
|
|
12468
|
-
|
|
12469
|
-
|
|
12470
|
-
|
|
12471
|
-
|
|
12472
|
-
|
|
12473
|
-
|
|
12474
|
-
|
|
12709
|
+
const persisted =
|
|
12710
|
+
this._coordinates.persistBackboneCoordinateFieldsNativeTransaction(
|
|
12711
|
+
{
|
|
12712
|
+
coordinateIndex: this.entryCoordinatesIndex as PutAndDeleteIndex<
|
|
12713
|
+
EntryReplicated<R>
|
|
12714
|
+
>,
|
|
12715
|
+
fields: coordinateFields,
|
|
12716
|
+
hash: facts.hash,
|
|
12717
|
+
deleteHashes: [],
|
|
12718
|
+
coordinates: backboneAppend.coordinate
|
|
12719
|
+
.coordinates as NumberFromType<R>[],
|
|
12720
|
+
skipGenericTransientCoordinateIndex: runtimeOnlyCoordinates,
|
|
12721
|
+
},
|
|
12722
|
+
ownershipLifecycleController,
|
|
12723
|
+
);
|
|
12475
12724
|
if (isPromiseLike(persisted)) {
|
|
12476
12725
|
await persisted;
|
|
12477
12726
|
this.throwIfReplicationOwnershipLifecycleInactive(
|
|
@@ -13982,6 +14231,8 @@ export class SharedLog<
|
|
|
13982
14231
|
this._lastLocalAppendAt = 0;
|
|
13983
14232
|
this._announcements ??= this.createReplicationAnnouncementCoordinator();
|
|
13984
14233
|
this._announcements.resetForOpen();
|
|
14234
|
+
this._v2Receive ??= this.createReplicationInfoV2ReceiveCoordinator();
|
|
14235
|
+
this._v2Receive.resetForOpen();
|
|
13985
14236
|
this._v2Send ??= this.createReplicationInfoV2SendCoordinator();
|
|
13986
14237
|
this._v2Send.resetForOpen();
|
|
13987
14238
|
const adaptiveReplicateOptions =
|
|
@@ -14725,8 +14976,11 @@ export class SharedLog<
|
|
|
14725
14976
|
// tombstone was written. Complete that erase before the adapter can expose
|
|
14726
14977
|
// any stale coordinate or document state to this backbone.
|
|
14727
14978
|
await this._coordinates._nativeBackboneCoordinatePersistence.resumeDrop?.();
|
|
14728
|
-
await this._coordinates._nativeBackboneCoordinatePersistence.hydrate(
|
|
14729
|
-
|
|
14979
|
+
await this._coordinates._nativeBackboneCoordinatePersistence.hydrate(
|
|
14980
|
+
backbone,
|
|
14981
|
+
);
|
|
14982
|
+
this._coordinates._nativeBackboneCoordinateJournalLastFlushMs =
|
|
14983
|
+
Date.now();
|
|
14730
14984
|
this.hydrateNativeCoordinateStateFromBackbone(backbone);
|
|
14731
14985
|
return;
|
|
14732
14986
|
}
|
|
@@ -14766,7 +15020,9 @@ export class SharedLog<
|
|
|
14766
15020
|
if (!this._nativeBackbone) {
|
|
14767
15021
|
return;
|
|
14768
15022
|
}
|
|
14769
|
-
const hashes = new Set(
|
|
15023
|
+
const hashes = new Set(
|
|
15024
|
+
this._coordinates._residentEntryCoordinatesByHash?.keys() ?? [],
|
|
15025
|
+
);
|
|
14770
15026
|
const iterator = this.entryCoordinatesIndex.iterate({});
|
|
14771
15027
|
try {
|
|
14772
15028
|
for (;;) {
|
|
@@ -14836,7 +15092,10 @@ export class SharedLog<
|
|
|
14836
15092
|
coordinate.requestedReplicas,
|
|
14837
15093
|
sharedFields.hashNumber,
|
|
14838
15094
|
);
|
|
14839
|
-
this._coordinates._residentEntryCoordinatesByHash.set(
|
|
15095
|
+
this._coordinates._residentEntryCoordinatesByHash.set(
|
|
15096
|
+
sharedFields.hash,
|
|
15097
|
+
sharedFields,
|
|
15098
|
+
);
|
|
14840
15099
|
for (const value of sharedFields.coordinates) {
|
|
14841
15100
|
this.coordinateToHash.add(value, sharedFields.hash);
|
|
14842
15101
|
}
|
|
@@ -14995,7 +15254,8 @@ export class SharedLog<
|
|
|
14995
15254
|
this._coordinates._nativeBackboneCoordinatePersistence
|
|
14996
15255
|
) {
|
|
14997
15256
|
if (
|
|
14998
|
-
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15257
|
+
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15258
|
+
.durableBarrier !== true ||
|
|
14999
15259
|
typeof this._nativeBackboneCoordinatePersistenceStore
|
|
15000
15260
|
?.durableBarrier !== "function"
|
|
15001
15261
|
) {
|
|
@@ -15006,11 +15266,12 @@ export class SharedLog<
|
|
|
15006
15266
|
}
|
|
15007
15267
|
if (
|
|
15008
15268
|
this._coordinates._nativeBackboneCoordinatePersistence &&
|
|
15009
|
-
(this._coordinates._nativeBackboneCoordinatePersistence
|
|
15010
|
-
null ||
|
|
15011
|
-
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15012
|
-
null) &&
|
|
15013
|
-
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15269
|
+
(this._coordinates._nativeBackboneCoordinatePersistence
|
|
15270
|
+
.compactMaxJournalBytes != null ||
|
|
15271
|
+
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15272
|
+
.compactMaxJournalRecords != null) &&
|
|
15273
|
+
this._coordinates._nativeBackboneCoordinatePersistence
|
|
15274
|
+
.crashSafeCompaction !== true
|
|
15014
15275
|
) {
|
|
15015
15276
|
// Durable custom adapters must explicitly advertise an atomic generation
|
|
15016
15277
|
// protocol before SharedLog permits automatic WAL compaction.
|
|
@@ -15236,7 +15497,8 @@ export class SharedLog<
|
|
|
15236
15497
|
async afterOpen(): Promise<void> {
|
|
15237
15498
|
await super.afterOpen();
|
|
15238
15499
|
const existingSubscribersPromise = this._getTopicSubscribers(this.topic);
|
|
15239
|
-
const replicationLifecycleController =
|
|
15500
|
+
const replicationLifecycleController =
|
|
15501
|
+
this._instanceLifecycle?.membershipLifecycleController;
|
|
15240
15502
|
|
|
15241
15503
|
// We do this here, because these calls requires this.closed == false
|
|
15242
15504
|
void this.pruneOfflineReplicators()
|
|
@@ -15277,7 +15539,8 @@ export class SharedLog<
|
|
|
15277
15539
|
async pruneOfflineReplicators() {
|
|
15278
15540
|
// Go through all segments and wait for replicators to become reachable;
|
|
15279
15541
|
// otherwise prune them away from the local membership view.
|
|
15280
|
-
const replicationLifecycleController =
|
|
15542
|
+
const replicationLifecycleController =
|
|
15543
|
+
this._instanceLifecycle?.membershipLifecycleController;
|
|
15281
15544
|
try {
|
|
15282
15545
|
if (
|
|
15283
15546
|
!replicationLifecycleController ||
|
|
@@ -15472,13 +15735,19 @@ export class SharedLog<
|
|
|
15472
15735
|
peerHash: string,
|
|
15473
15736
|
ownershipLifecycleController = this.captureReplicationOwnershipLifecycle(),
|
|
15474
15737
|
) {
|
|
15738
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
15739
|
+
const preserveV2Session =
|
|
15740
|
+
peerSession?.phase === "open" && peerSession.isActive();
|
|
15475
15741
|
this.cancelReplicationInfoRequests(peerHash);
|
|
15476
15742
|
this._liveness._replicatorLivenessFailures.delete(peerHash);
|
|
15477
15743
|
this._liveness._replicatorLastActivityAt.delete(peerHash);
|
|
15478
|
-
|
|
15479
|
-
|
|
15480
|
-
|
|
15481
|
-
|
|
15744
|
+
if (!preserveV2Session) {
|
|
15745
|
+
this._peerSyncCapabilities.delete(peerHash);
|
|
15746
|
+
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
15747
|
+
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
15748
|
+
this._v2Receive.clearPeer(peerHash);
|
|
15749
|
+
this._v2Send.clearPeer(peerHash);
|
|
15750
|
+
}
|
|
15482
15751
|
this.cleanupPendingIHavePeer(peerHash);
|
|
15483
15752
|
this.cleanupCheckedPrunePeer(
|
|
15484
15753
|
peerHash,
|
|
@@ -15502,8 +15771,16 @@ export class SharedLog<
|
|
|
15502
15771
|
// when they eventually reach the apply lane. Reset the sender's
|
|
15503
15772
|
// ordering watermark with the local epoch so a later arrival can be
|
|
15504
15773
|
// accepted without comparing its clock to this receiver's clock.
|
|
15505
|
-
this._peerSessions.advanceReceiveEpoch(peerHash);
|
|
15774
|
+
const receiveEpoch = this._peerSessions.advanceReceiveEpoch(peerHash);
|
|
15506
15775
|
this.latestReplicationInfoMessage.delete(peerHash);
|
|
15776
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
15777
|
+
if (peerSession?.phase === "open") {
|
|
15778
|
+
this._v2Receive.advanceRecovery({
|
|
15779
|
+
peerHash,
|
|
15780
|
+
peerSession,
|
|
15781
|
+
receiveEpoch,
|
|
15782
|
+
});
|
|
15783
|
+
}
|
|
15507
15784
|
}
|
|
15508
15785
|
|
|
15509
15786
|
private async resolveCandidatePeersForHash(
|
|
@@ -15784,7 +16061,9 @@ export class SharedLog<
|
|
|
15784
16061
|
deferredCoordinateDeleteHashes,
|
|
15785
16062
|
);
|
|
15786
16063
|
} else {
|
|
15787
|
-
this._coordinates.forgetCoordinateStateForHashes(
|
|
16064
|
+
this._coordinates.forgetCoordinateStateForHashes(
|
|
16065
|
+
deferredCoordinateDeleteHashes,
|
|
16066
|
+
);
|
|
15788
16067
|
}
|
|
15789
16068
|
return deferredCoordinateDeleteHashes;
|
|
15790
16069
|
}
|
|
@@ -15832,7 +16111,9 @@ export class SharedLog<
|
|
|
15832
16111
|
deferredCoordinateDeleteHashes,
|
|
15833
16112
|
);
|
|
15834
16113
|
} else {
|
|
15835
|
-
this._coordinates.forgetCoordinateStateForHashes(
|
|
16114
|
+
this._coordinates.forgetCoordinateStateForHashes(
|
|
16115
|
+
deferredCoordinateDeleteHashes,
|
|
16116
|
+
);
|
|
15836
16117
|
}
|
|
15837
16118
|
return deferredCoordinateDeleteHashes;
|
|
15838
16119
|
}
|
|
@@ -16395,7 +16676,9 @@ export class SharedLog<
|
|
|
16395
16676
|
this._wireSyncSession = undefined;
|
|
16396
16677
|
}
|
|
16397
16678
|
});
|
|
16398
|
-
await capture(() =>
|
|
16679
|
+
await capture(() =>
|
|
16680
|
+
this._coordinates.closeNativeBackboneCoordinatePersistence(),
|
|
16681
|
+
);
|
|
16399
16682
|
await capture(() => this.syncronizer?.close());
|
|
16400
16683
|
|
|
16401
16684
|
captureSync(() => {
|
|
@@ -16490,6 +16773,7 @@ export class SharedLog<
|
|
|
16490
16773
|
this._peerSyncCapabilities?.clear();
|
|
16491
16774
|
this._peerSyncCapabilitySessions?.clear();
|
|
16492
16775
|
this._peerSyncCapabilityTimestamps?.clear();
|
|
16776
|
+
this._v2Receive?.clearForClose();
|
|
16493
16777
|
this._v2Send?.clearForClose();
|
|
16494
16778
|
this._liveRawGossipBatches?.clear();
|
|
16495
16779
|
this._nativeSharedLogState?.clearGidPeers();
|
|
@@ -16563,6 +16847,7 @@ export class SharedLog<
|
|
|
16563
16847
|
this.preventParentAttachments();
|
|
16564
16848
|
this.stopRepairLifecycle();
|
|
16565
16849
|
this._instanceLifecycle?.beginTerminal("close");
|
|
16850
|
+
this._v2Receive?.clearForClose();
|
|
16566
16851
|
const replicationRangeTerminalFence =
|
|
16567
16852
|
this.acquireReplicationRangeMutationTerminalFence();
|
|
16568
16853
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
@@ -16625,10 +16910,12 @@ export class SharedLog<
|
|
|
16625
16910
|
try {
|
|
16626
16911
|
const reset = new AllReplicatingSegmentsMessage({ segments: [] });
|
|
16627
16912
|
await Promise.all([
|
|
16628
|
-
this.rpc
|
|
16629
|
-
|
|
16630
|
-
|
|
16631
|
-
|
|
16913
|
+
this.rpc
|
|
16914
|
+
.send(reset, {
|
|
16915
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
16916
|
+
signal: abort.signal,
|
|
16917
|
+
})
|
|
16918
|
+
.catch(() => {}),
|
|
16632
16919
|
this._v2Send.sendTerminalReset(abort.signal),
|
|
16633
16920
|
]);
|
|
16634
16921
|
} finally {
|
|
@@ -16688,7 +16975,8 @@ export class SharedLog<
|
|
|
16688
16975
|
}
|
|
16689
16976
|
this.throwIfCheckedPruneRemoveBlocksLocalOperation("drop");
|
|
16690
16977
|
this.ensureNativeDurabilityRuntimeState();
|
|
16691
|
-
const nativePersistence =
|
|
16978
|
+
const nativePersistence =
|
|
16979
|
+
this._coordinates._nativeBackboneCoordinatePersistence;
|
|
16692
16980
|
if (
|
|
16693
16981
|
nativePersistence &&
|
|
16694
16982
|
(typeof nativePersistence.drop !== "function" ||
|
|
@@ -16706,6 +16994,7 @@ export class SharedLog<
|
|
|
16706
16994
|
this.preventParentAttachments();
|
|
16707
16995
|
this.stopRepairLifecycle();
|
|
16708
16996
|
this._instanceLifecycle?.beginTerminal("drop");
|
|
16997
|
+
this._v2Receive?.clearForClose();
|
|
16709
16998
|
const replicationRangeTerminalFence =
|
|
16710
16999
|
this.acquireReplicationRangeMutationTerminalFence();
|
|
16711
17000
|
const pruneRemoveTerminalFence = this.acquirePruneRemoveTerminalFence();
|
|
@@ -16751,10 +17040,12 @@ export class SharedLog<
|
|
|
16751
17040
|
try {
|
|
16752
17041
|
const reset = new AllReplicatingSegmentsMessage({ segments: [] });
|
|
16753
17042
|
await Promise.all([
|
|
16754
|
-
this.rpc
|
|
16755
|
-
|
|
16756
|
-
|
|
16757
|
-
|
|
17043
|
+
this.rpc
|
|
17044
|
+
.send(reset, {
|
|
17045
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
17046
|
+
signal: abort.signal,
|
|
17047
|
+
})
|
|
17048
|
+
.catch(() => {}),
|
|
16758
17049
|
this._v2Send.sendTerminalReset(abort.signal),
|
|
16759
17050
|
]);
|
|
16760
17051
|
} finally {
|
|
@@ -17273,13 +17564,6 @@ export class SharedLog<
|
|
|
17273
17564
|
msg: TransportMessage,
|
|
17274
17565
|
context: RequestContext,
|
|
17275
17566
|
): Promise<void> {
|
|
17276
|
-
// Decode-first only: capability advertisement is not permission to use
|
|
17277
|
-
// V2. Drop every unsolicited V2 envelope before even entering the shared
|
|
17278
|
-
// receive envelope, so durable poison checks, leases, liveness, ordering
|
|
17279
|
-
// watermarks, replication ranges and cleanup side effects cannot observe it.
|
|
17280
|
-
if (isReplicationInfoV2Message(msg)) {
|
|
17281
|
-
return;
|
|
17282
|
-
}
|
|
17283
17567
|
const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
|
|
17284
17568
|
? msg
|
|
17285
17569
|
: undefined;
|
|
@@ -17332,19 +17616,28 @@ export class SharedLog<
|
|
|
17332
17616
|
}
|
|
17333
17617
|
if (
|
|
17334
17618
|
msg instanceof AllReplicatingSegmentsMessage ||
|
|
17335
|
-
msg instanceof AddedReplicationSegmentMessage
|
|
17619
|
+
msg instanceof AddedReplicationSegmentMessage ||
|
|
17620
|
+
msg instanceof FullReplicationInfoV2Message ||
|
|
17621
|
+
msg instanceof AddedReplicationInfoV2Message
|
|
17336
17622
|
) {
|
|
17337
17623
|
// Bound decoded untrusted vectors before per-peer/global mutation
|
|
17338
17624
|
// queues, trusted-replicator authorization, or liveness side effects.
|
|
17339
17625
|
this.validateReplicationRangeAnnouncement(msg.segments);
|
|
17340
|
-
} else if (
|
|
17626
|
+
} else if (
|
|
17627
|
+
msg instanceof StoppedReplicating ||
|
|
17628
|
+
msg instanceof StoppedReplicationInfoV2Message
|
|
17629
|
+
) {
|
|
17341
17630
|
// Bound the raw decoded vector before deduplication can hide the
|
|
17342
17631
|
// allocation cost, and before liveness or apply-queue side effects.
|
|
17343
17632
|
this.validateStoppedReplicationAnnouncement(msg.segmentIds);
|
|
17344
17633
|
}
|
|
17345
17634
|
if (
|
|
17346
17635
|
!context.from.equals(this.node.identity.publicKey) &&
|
|
17347
|
-
!(msg instanceof RequestReplicationInfoV2Message)
|
|
17636
|
+
!(msg instanceof RequestReplicationInfoV2Message) &&
|
|
17637
|
+
!isReplicationInfoV2Message(msg) &&
|
|
17638
|
+
!(msg instanceof AllReplicatingSegmentsMessage) &&
|
|
17639
|
+
!(msg instanceof AddedReplicationSegmentMessage) &&
|
|
17640
|
+
!(msg instanceof StoppedReplicating)
|
|
17348
17641
|
) {
|
|
17349
17642
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
17350
17643
|
}
|
|
@@ -18390,11 +18683,14 @@ export class SharedLog<
|
|
|
18390
18683
|
);
|
|
18391
18684
|
}
|
|
18392
18685
|
const reusableCoordinatePlans =
|
|
18393
|
-
this._coordinates.createReusableReceiveCoordinatePlans(
|
|
18394
|
-
|
|
18395
|
-
|
|
18396
|
-
|
|
18397
|
-
|
|
18686
|
+
this._coordinates.createReusableReceiveCoordinatePlans(
|
|
18687
|
+
receiveGroups,
|
|
18688
|
+
{
|
|
18689
|
+
decodedReplicaCounts: receiveReplicaCounts,
|
|
18690
|
+
allowRoleAgeZeroPlans:
|
|
18691
|
+
immediateReplicatingLeaderPlans !== undefined,
|
|
18692
|
+
},
|
|
18693
|
+
) as Map<string, ReusableReceiveCoordinatePlan<R>>;
|
|
18398
18694
|
if (syncProfile) {
|
|
18399
18695
|
emitSyncProfileDuration(syncProfile, joinPlanStartedAt, {
|
|
18400
18696
|
name: "sharedLog.receive.joinPlan",
|
|
@@ -18789,9 +19085,7 @@ export class SharedLog<
|
|
|
18789
19085
|
}
|
|
18790
19086
|
const checkedPruneStartedAt = syncProfileStart(syncProfile);
|
|
18791
19087
|
const ownershipChangedDuringReceive =
|
|
18792
|
-
!this.isReceiveOwnershipSnapshotStable(
|
|
18793
|
-
receiveOwnershipRevision,
|
|
18794
|
-
);
|
|
19088
|
+
!this.isReceiveOwnershipSnapshotStable(receiveOwnershipRevision);
|
|
18795
19089
|
if (ownershipChangedDuringReceive) {
|
|
18796
19090
|
const freshAuditRevision =
|
|
18797
19091
|
this._instanceLifecycle?._receiveOwnershipRevision ?? 0;
|
|
@@ -18949,6 +19243,7 @@ export class SharedLog<
|
|
|
18949
19243
|
fromHash: receiveFromHash,
|
|
18950
19244
|
session: receiveSession,
|
|
18951
19245
|
lifecycleController: receiveReplicationLifecycleController,
|
|
19246
|
+
ownershipLifecycleController: receiveOwnershipLifecycleController,
|
|
18952
19247
|
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
18953
19248
|
syncProfile,
|
|
18954
19249
|
lease: peerReceiveLease,
|
|
@@ -18972,6 +19267,12 @@ export class SharedLog<
|
|
|
18972
19267
|
this.markEntriesKnownByPeer(msg.hashes, context.from.hashcode());
|
|
18973
19268
|
this.clearRepairFrontierHashes(context.from.hashcode(), msg.hashes);
|
|
18974
19269
|
return;
|
|
19270
|
+
} else if (isReplicationInfoV2Message(msg)) {
|
|
19271
|
+
await this.handleReplicationInfoV2Announcement(
|
|
19272
|
+
msg,
|
|
19273
|
+
laneRequestContext,
|
|
19274
|
+
lane,
|
|
19275
|
+
);
|
|
18975
19276
|
} else if (msg instanceof SyncCapabilitiesMessage) {
|
|
18976
19277
|
if (!context.from.equals(this.node.identity.publicKey)) {
|
|
18977
19278
|
const capabilityTransportSession = context.message?.header?.session;
|
|
@@ -18993,13 +19294,19 @@ export class SharedLog<
|
|
|
18993
19294
|
timestamp: capabilityTimestamp,
|
|
18994
19295
|
openingSession: receiveSession!,
|
|
18995
19296
|
});
|
|
18996
|
-
} else
|
|
19297
|
+
} else if (
|
|
18997
19298
|
this.observePeerSyncCapabilities({
|
|
18998
19299
|
peerHash: receiveFromHash,
|
|
18999
19300
|
capabilities: msg.capabilities,
|
|
19000
19301
|
transportSession: capabilityTransportSession,
|
|
19001
19302
|
timestamp: capabilityTimestamp,
|
|
19002
|
-
})
|
|
19303
|
+
}) &&
|
|
19304
|
+
receiveSession?.phase === "open"
|
|
19305
|
+
) {
|
|
19306
|
+
this.promoteReplicationInfoV2ReceiveCapability(
|
|
19307
|
+
context.from,
|
|
19308
|
+
receiveSession,
|
|
19309
|
+
);
|
|
19003
19310
|
}
|
|
19004
19311
|
}
|
|
19005
19312
|
return;
|
|
@@ -19021,6 +19328,8 @@ export class SharedLog<
|
|
|
19021
19328
|
from: context.from,
|
|
19022
19329
|
peerSession: receiveSession,
|
|
19023
19330
|
receiverTransportSession: context.message.header.session,
|
|
19331
|
+
capabilityTimestamp:
|
|
19332
|
+
this._peerSyncCapabilityTimestamps.get(receiveFromHash)!,
|
|
19024
19333
|
requestTimestamp: context.message.header.timestamp,
|
|
19025
19334
|
})
|
|
19026
19335
|
) {
|
|
@@ -19164,9 +19473,7 @@ export class SharedLog<
|
|
|
19164
19473
|
msg.requests,
|
|
19165
19474
|
);
|
|
19166
19475
|
if (
|
|
19167
|
-
!this.isReplicationLifecycleActive(
|
|
19168
|
-
receiveReplicationLifecycleController,
|
|
19169
|
-
)
|
|
19476
|
+
!this.isReplicationLifecycleActive(receiveReplicationLifecycleController)
|
|
19170
19477
|
) {
|
|
19171
19478
|
return;
|
|
19172
19479
|
}
|
|
@@ -19225,9 +19532,7 @@ export class SharedLog<
|
|
|
19225
19532
|
let entry: Entry<T> | undefined;
|
|
19226
19533
|
try {
|
|
19227
19534
|
if (await this.log.blocks.has(request.hash)) {
|
|
19228
|
-
entry = (await this.log.get(request.hash)) as
|
|
19229
|
-
| Entry<T>
|
|
19230
|
-
| undefined;
|
|
19535
|
+
entry = (await this.log.get(request.hash)) as Entry<T> | undefined;
|
|
19231
19536
|
}
|
|
19232
19537
|
} catch {
|
|
19233
19538
|
// The normal entry-admission hook will retry this path.
|
|
@@ -19268,16 +19573,11 @@ export class SharedLog<
|
|
|
19268
19573
|
}
|
|
19269
19574
|
const responseTasks: Promise<void>[] = [];
|
|
19270
19575
|
for (const request of msg.requests) {
|
|
19271
|
-
const pendingDelete = this._checkedPrune.getPendingDelete(
|
|
19272
|
-
request.hash,
|
|
19273
|
-
);
|
|
19576
|
+
const pendingDelete = this._checkedPrune.getPendingDelete(request.hash);
|
|
19274
19577
|
if (pendingDelete) {
|
|
19275
19578
|
responseTasks.push(
|
|
19276
19579
|
Promise.resolve(
|
|
19277
|
-
pendingDelete.resolve(
|
|
19278
|
-
context.from.hashcode(),
|
|
19279
|
-
request.requestId,
|
|
19280
|
-
),
|
|
19580
|
+
pendingDelete.resolve(context.from.hashcode(), request.requestId),
|
|
19281
19581
|
),
|
|
19282
19582
|
);
|
|
19283
19583
|
}
|
|
@@ -19397,6 +19697,191 @@ export class SharedLog<
|
|
|
19397
19697
|
}
|
|
19398
19698
|
}
|
|
19399
19699
|
|
|
19700
|
+
private async handleReplicationInfoV2Announcement(
|
|
19701
|
+
msg: ReplicationInfoV2Message,
|
|
19702
|
+
context: ReceiveRequestContext,
|
|
19703
|
+
lane: ReceiveLaneContext,
|
|
19704
|
+
): Promise<void> {
|
|
19705
|
+
const from = context.from;
|
|
19706
|
+
const fromHash = lane.fromHash;
|
|
19707
|
+
const receiveSession = lane.session;
|
|
19708
|
+
if (
|
|
19709
|
+
from.equals(this.node.identity.publicKey) ||
|
|
19710
|
+
receiveSession === null ||
|
|
19711
|
+
receiveSession.phase !== "open"
|
|
19712
|
+
) {
|
|
19713
|
+
return;
|
|
19714
|
+
}
|
|
19715
|
+
|
|
19716
|
+
// Authenticate and reserve before the per-peer lane. One reservation per
|
|
19717
|
+
// peer bounds decoded-frame retention while another mutation is parked.
|
|
19718
|
+
const receiveState = this._v2Receive._receiveStates.get(fromHash);
|
|
19719
|
+
if (
|
|
19720
|
+
!receiveState ||
|
|
19721
|
+
receiveState.peerSession !== receiveSession ||
|
|
19722
|
+
receiveState.senderTransportSession !== context.message.header.session
|
|
19723
|
+
) {
|
|
19724
|
+
return;
|
|
19725
|
+
}
|
|
19726
|
+
const admission = this._v2Receive.reserve(msg, {
|
|
19727
|
+
from,
|
|
19728
|
+
peerSession: receiveSession,
|
|
19729
|
+
receiveEpoch: lane.receiveEpoch,
|
|
19730
|
+
senderTransportSession: context.message.header.session,
|
|
19731
|
+
transportTimestamp: context.message.header.timestamp,
|
|
19732
|
+
});
|
|
19733
|
+
if (!admission) {
|
|
19734
|
+
return;
|
|
19735
|
+
}
|
|
19736
|
+
|
|
19737
|
+
lane.lease.release();
|
|
19738
|
+
try {
|
|
19739
|
+
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
19740
|
+
const hostGate = () =>
|
|
19741
|
+
this._instanceLifecycle!.isMembershipActiveFor(
|
|
19742
|
+
lane.lifecycleController,
|
|
19743
|
+
) &&
|
|
19744
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController) &&
|
|
19745
|
+
this._peerSessions.isCurrent(fromHash, receiveSession) &&
|
|
19746
|
+
receiveSession.phase === "open" &&
|
|
19747
|
+
this._peerSessions.isReceiveEpochCurrent(
|
|
19748
|
+
fromHash,
|
|
19749
|
+
lane.receiveEpoch,
|
|
19750
|
+
) &&
|
|
19751
|
+
!this._peerSessions.isReplicationInfoBlocked(fromHash) &&
|
|
19752
|
+
this._peerSessions.isReceiveCleanupGateOpen(fromHash);
|
|
19753
|
+
if (!hostGate()) {
|
|
19754
|
+
return;
|
|
19755
|
+
}
|
|
19756
|
+
const exactGate = () =>
|
|
19757
|
+
hostGate() && this._v2Receive.isAdmissionCurrent(admission);
|
|
19758
|
+
let durableCommitted = false;
|
|
19759
|
+
|
|
19760
|
+
try {
|
|
19761
|
+
if (
|
|
19762
|
+
msg instanceof FullReplicationInfoV2Message ||
|
|
19763
|
+
msg instanceof AddedReplicationInfoV2Message
|
|
19764
|
+
) {
|
|
19765
|
+
let mutationGateChecked = false;
|
|
19766
|
+
let mutationGateAdmitted = false;
|
|
19767
|
+
const result = await this.addReplicationRange(
|
|
19768
|
+
msg.segments.map((segment) =>
|
|
19769
|
+
segment.toReplicationRangeIndexable(from),
|
|
19770
|
+
),
|
|
19771
|
+
from,
|
|
19772
|
+
{
|
|
19773
|
+
reset: msg instanceof FullReplicationInfoV2Message,
|
|
19774
|
+
checkDuplicates: true,
|
|
19775
|
+
timestamp: Number(context.message.header.timestamp),
|
|
19776
|
+
allowLegacyOrderedReplacementPairs:
|
|
19777
|
+
msg instanceof AddedReplicationInfoV2Message,
|
|
19778
|
+
shouldApply: () => {
|
|
19779
|
+
mutationGateChecked = true;
|
|
19780
|
+
mutationGateAdmitted = exactGate();
|
|
19781
|
+
return mutationGateAdmitted;
|
|
19782
|
+
},
|
|
19783
|
+
onDurableApplyCommitted: () => {
|
|
19784
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
19785
|
+
return false;
|
|
19786
|
+
}
|
|
19787
|
+
durableCommitted = true;
|
|
19788
|
+
return true;
|
|
19789
|
+
},
|
|
19790
|
+
},
|
|
19791
|
+
lane.ownershipLifecycleController,
|
|
19792
|
+
);
|
|
19793
|
+
if (
|
|
19794
|
+
result === undefined ||
|
|
19795
|
+
!mutationGateChecked ||
|
|
19796
|
+
!mutationGateAdmitted ||
|
|
19797
|
+
!durableCommitted ||
|
|
19798
|
+
!exactGate()
|
|
19799
|
+
) {
|
|
19800
|
+
return;
|
|
19801
|
+
}
|
|
19802
|
+
} else {
|
|
19803
|
+
if (!exactGate()) {
|
|
19804
|
+
return;
|
|
19805
|
+
}
|
|
19806
|
+
const rangesToRemove =
|
|
19807
|
+
await this.resolveReplicationRangesFromIdsAndKey(
|
|
19808
|
+
msg.segmentIds,
|
|
19809
|
+
from,
|
|
19810
|
+
);
|
|
19811
|
+
if (!exactGate()) {
|
|
19812
|
+
return;
|
|
19813
|
+
}
|
|
19814
|
+
let mutationGateAdmitted = true;
|
|
19815
|
+
const removedRanges: ReplicationRangeIndexable<R>[] = [];
|
|
19816
|
+
const removed = await this.removeReplicationRanges(
|
|
19817
|
+
rangesToRemove,
|
|
19818
|
+
from,
|
|
19819
|
+
{
|
|
19820
|
+
shouldRemove: () => {
|
|
19821
|
+
mutationGateAdmitted = exactGate();
|
|
19822
|
+
return mutationGateAdmitted;
|
|
19823
|
+
},
|
|
19824
|
+
onDurableRemoveCommitted: () => {
|
|
19825
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
19826
|
+
return false;
|
|
19827
|
+
}
|
|
19828
|
+
durableCommitted = true;
|
|
19829
|
+
return true;
|
|
19830
|
+
},
|
|
19831
|
+
onRemoved: (ranges) => removedRanges.push(...ranges),
|
|
19832
|
+
},
|
|
19833
|
+
lane.ownershipLifecycleController,
|
|
19834
|
+
);
|
|
19835
|
+
if (!durableCommitted) {
|
|
19836
|
+
if (
|
|
19837
|
+
!mutationGateAdmitted ||
|
|
19838
|
+
!exactGate() ||
|
|
19839
|
+
removed ||
|
|
19840
|
+
!this._v2Receive.commit(admission)
|
|
19841
|
+
) {
|
|
19842
|
+
return;
|
|
19843
|
+
}
|
|
19844
|
+
durableCommitted = true;
|
|
19845
|
+
}
|
|
19846
|
+
if (
|
|
19847
|
+
this._instanceLifecycle!.isMembershipActiveFor(
|
|
19848
|
+
lane.lifecycleController,
|
|
19849
|
+
) &&
|
|
19850
|
+
this.isRepairLifecycleActive(lane.ownershipLifecycleController)
|
|
19851
|
+
) {
|
|
19852
|
+
const timestamp = BigInt(Date.now());
|
|
19853
|
+
for (const range of removedRanges) {
|
|
19854
|
+
this.replicationChangeDebounceFn.add({
|
|
19855
|
+
range,
|
|
19856
|
+
type: "removed",
|
|
19857
|
+
timestamp,
|
|
19858
|
+
});
|
|
19859
|
+
}
|
|
19860
|
+
}
|
|
19861
|
+
}
|
|
19862
|
+
} catch (error) {
|
|
19863
|
+
this._v2Receive.requireFullAfterFailure(admission);
|
|
19864
|
+
throw error;
|
|
19865
|
+
}
|
|
19866
|
+
|
|
19867
|
+
if (!durableCommitted) {
|
|
19868
|
+
if (!exactGate() || !this._v2Receive.commit(admission)) {
|
|
19869
|
+
return;
|
|
19870
|
+
}
|
|
19871
|
+
}
|
|
19872
|
+
if (!hostGate()) {
|
|
19873
|
+
return;
|
|
19874
|
+
}
|
|
19875
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
19876
|
+
if (msg instanceof FullReplicationInfoV2Message) {
|
|
19877
|
+
this.cancelReplicationInfoRequests(fromHash);
|
|
19878
|
+
}
|
|
19879
|
+
});
|
|
19880
|
+
} finally {
|
|
19881
|
+
this._v2Receive.release(admission);
|
|
19882
|
+
}
|
|
19883
|
+
}
|
|
19884
|
+
|
|
19400
19885
|
private async handleReplicationInfoAnnouncement(
|
|
19401
19886
|
msg: AllReplicatingSegmentsMessage | AddedReplicationSegmentMessage,
|
|
19402
19887
|
context: ReceiveRequestContext,
|
|
@@ -19420,6 +19905,19 @@ export class SharedLog<
|
|
|
19420
19905
|
// (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
|
|
19421
19906
|
const from = context.from!;
|
|
19422
19907
|
const fromHash = from.hashcode();
|
|
19908
|
+
if (this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
19909
|
+
if (receiveSession) {
|
|
19910
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
19911
|
+
peerHash: fromHash,
|
|
19912
|
+
peerSession: receiveSession,
|
|
19913
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
19914
|
+
senderTransportSession: context.message.header.session,
|
|
19915
|
+
transportTimestamp: context.message.header.timestamp,
|
|
19916
|
+
message: msg,
|
|
19917
|
+
});
|
|
19918
|
+
}
|
|
19919
|
+
return;
|
|
19920
|
+
}
|
|
19423
19921
|
// Pre-lane gate: lifecycle -> receive-epoch -> blocked, exactly the
|
|
19424
19922
|
// legacy order. isMembershipActiveFor is the unit-pinned fold of
|
|
19425
19923
|
// isReplicationLifecycleActive; isReceiveEpochCurrent is the
|
|
@@ -19458,6 +19956,17 @@ export class SharedLog<
|
|
|
19458
19956
|
) {
|
|
19459
19957
|
return;
|
|
19460
19958
|
}
|
|
19959
|
+
if (receiveSession && this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
19960
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
19961
|
+
peerHash: fromHash,
|
|
19962
|
+
peerSession: receiveSession,
|
|
19963
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
19964
|
+
senderTransportSession: context.message.header.session,
|
|
19965
|
+
transportTimestamp: context.message.header.timestamp,
|
|
19966
|
+
message: msg,
|
|
19967
|
+
});
|
|
19968
|
+
return;
|
|
19969
|
+
}
|
|
19461
19970
|
|
|
19462
19971
|
// Process in-order to avoid races where repeated reset messages arrive
|
|
19463
19972
|
// concurrently and trigger spurious "added" diffs / rebalancing.
|
|
@@ -19467,14 +19976,13 @@ export class SharedLog<
|
|
|
19467
19976
|
}
|
|
19468
19977
|
|
|
19469
19978
|
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
19470
|
-
this._liveness._replicatorLivenessFailures.delete(fromHash);
|
|
19471
19979
|
|
|
19472
19980
|
if (this.closed) {
|
|
19473
19981
|
return;
|
|
19474
19982
|
}
|
|
19475
19983
|
|
|
19476
19984
|
const reset = msg instanceof AllReplicatingSegmentsMessage;
|
|
19477
|
-
await this.addReplicationRange(
|
|
19985
|
+
const result = await this.addReplicationRange(
|
|
19478
19986
|
replicationInfoMessage.segments.map((x) =>
|
|
19479
19987
|
x.toReplicationRangeIndexable(from),
|
|
19480
19988
|
),
|
|
@@ -19487,6 +19995,10 @@ export class SharedLog<
|
|
|
19487
19995
|
msg instanceof AddedReplicationSegmentMessage,
|
|
19488
19996
|
},
|
|
19489
19997
|
);
|
|
19998
|
+
if (result === undefined) {
|
|
19999
|
+
return;
|
|
20000
|
+
}
|
|
20001
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
19490
20002
|
|
|
19491
20003
|
// If the peer reports any replication segments, stop re-requesting.
|
|
19492
20004
|
// (Empty reports can be transient during startup.)
|
|
@@ -19522,6 +20034,19 @@ export class SharedLog<
|
|
|
19522
20034
|
return;
|
|
19523
20035
|
}
|
|
19524
20036
|
const fromHash = from.hashcode();
|
|
20037
|
+
if (this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
20038
|
+
if (receiveSession) {
|
|
20039
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
20040
|
+
peerHash: fromHash,
|
|
20041
|
+
peerSession: receiveSession,
|
|
20042
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
20043
|
+
senderTransportSession: context.message.header.session,
|
|
20044
|
+
transportTimestamp: context.message.header.timestamp,
|
|
20045
|
+
message: msg,
|
|
20046
|
+
});
|
|
20047
|
+
}
|
|
20048
|
+
return;
|
|
20049
|
+
}
|
|
19525
20050
|
// Same pre-lane gate shape as Added/All above (and the same
|
|
19526
20051
|
// intentional absence of a subscription-epoch term).
|
|
19527
20052
|
if (
|
|
@@ -19552,22 +20077,34 @@ export class SharedLog<
|
|
|
19552
20077
|
) {
|
|
19553
20078
|
return;
|
|
19554
20079
|
}
|
|
20080
|
+
if (receiveSession && this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
20081
|
+
this._v2Receive.noteLegacyAnnouncement({
|
|
20082
|
+
peerHash: fromHash,
|
|
20083
|
+
peerSession: receiveSession,
|
|
20084
|
+
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
20085
|
+
senderTransportSession: context.message.header.session,
|
|
20086
|
+
transportTimestamp: context.message.header.timestamp,
|
|
20087
|
+
message: msg,
|
|
20088
|
+
});
|
|
20089
|
+
return;
|
|
20090
|
+
}
|
|
19555
20091
|
|
|
19556
|
-
const previousTimestamp =
|
|
19557
|
-
this.latestReplicationInfoMessage.get(fromHash);
|
|
20092
|
+
const previousTimestamp = this.latestReplicationInfoMessage.get(fromHash);
|
|
19558
20093
|
if (previousTimestamp && previousTimestamp > messageTimestamp) {
|
|
19559
20094
|
return;
|
|
19560
20095
|
}
|
|
19561
20096
|
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
19562
|
-
this._liveness._replicatorLivenessFailures.delete(fromHash);
|
|
19563
20097
|
if (this.closed) {
|
|
19564
20098
|
return;
|
|
19565
20099
|
}
|
|
19566
20100
|
|
|
19567
|
-
const rangesToRemove =
|
|
19568
|
-
|
|
20101
|
+
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
|
|
20102
|
+
segmentIds,
|
|
20103
|
+
from,
|
|
20104
|
+
);
|
|
19569
20105
|
|
|
19570
20106
|
await this.removeReplicationRanges(rangesToRemove, from);
|
|
20107
|
+
this._liveness.markReplicatorActivity(fromHash);
|
|
19571
20108
|
const timestamp = BigInt(+new Date());
|
|
19572
20109
|
for (const range of rangesToRemove) {
|
|
19573
20110
|
this.replicationChangeDebounceFn.add({
|
|
@@ -20369,10 +20906,9 @@ export class SharedLog<
|
|
|
20369
20906
|
entry: ShallowOrFullEntry<any> | EntryReplicated<R> | NumberFromType<R>,
|
|
20370
20907
|
minReplicas: number,
|
|
20371
20908
|
): Promise<NumberFromType<R>[]> {
|
|
20372
|
-
return this._coordinates.createCoordinates(
|
|
20373
|
-
|
|
20374
|
-
|
|
20375
|
-
) as Promise<NumberFromType<R>[]>;
|
|
20909
|
+
return this._coordinates.createCoordinates(entry, minReplicas) as Promise<
|
|
20910
|
+
NumberFromType<R>[]
|
|
20911
|
+
>;
|
|
20376
20912
|
}
|
|
20377
20913
|
|
|
20378
20914
|
async getDefaultMinRoleAge(): Promise<number> {
|
|
@@ -20736,8 +21272,9 @@ export class SharedLog<
|
|
|
20736
21272
|
this.createNativeLeaderOptions(context, firstItem.options),
|
|
20737
21273
|
);
|
|
20738
21274
|
const plans: EntryLeaderPlan<R>[] = [];
|
|
20739
|
-
const persistItems: Parameters<
|
|
20740
|
-
|
|
21275
|
+
const persistItems: Parameters<
|
|
21276
|
+
typeof this._coordinates.persistCoordinatesBatch
|
|
21277
|
+
>[0] = [];
|
|
20741
21278
|
for (let i = 0; i < itemArray.length; i++) {
|
|
20742
21279
|
const item = itemArray[i]!;
|
|
20743
21280
|
const nativePlan = nativePlans[i]!;
|
|
@@ -20799,8 +21336,9 @@ export class SharedLog<
|
|
|
20799
21336
|
);
|
|
20800
21337
|
const selfHash = this.node.identity.publicKey.hashcode();
|
|
20801
21338
|
const plans: EntryLeaderPlan<R>[] = [];
|
|
20802
|
-
const persistItems: Parameters<
|
|
20803
|
-
|
|
21339
|
+
const persistItems: Parameters<
|
|
21340
|
+
typeof this._coordinates.persistCoordinatesBatch
|
|
21341
|
+
>[0] = [];
|
|
20804
21342
|
for (let i = 0; i < itemArray.length; i++) {
|
|
20805
21343
|
const item = itemArray[i]!;
|
|
20806
21344
|
const nativePlan = nativePlans[i]!;
|
|
@@ -22411,6 +22949,8 @@ export class SharedLog<
|
|
|
22411
22949
|
ownerHash: string,
|
|
22412
22950
|
options?: { preserveOwnerMembership?: boolean },
|
|
22413
22951
|
): Promise<ReplicationRangeDeletionOutcome<R>> {
|
|
22952
|
+
const wasReplicator = this.uniqueReplicators.has(ownerHash);
|
|
22953
|
+
const joinWasEmitted = this._replicatorJoinEmitted.has(ownerHash);
|
|
22414
22954
|
if (ranges.length === 0) {
|
|
22415
22955
|
const ownerHasRanges =
|
|
22416
22956
|
(await this.replicationIndex.count({ query: { hash: ownerHash } })) > 0;
|
|
@@ -22422,14 +22962,24 @@ export class SharedLog<
|
|
|
22422
22962
|
removed: [],
|
|
22423
22963
|
retained: [],
|
|
22424
22964
|
ownerHasRanges,
|
|
22965
|
+
rollback: async () => {
|
|
22966
|
+
if (wasReplicator) {
|
|
22967
|
+
this.uniqueReplicators.add(ownerHash);
|
|
22968
|
+
} else {
|
|
22969
|
+
this.uniqueReplicators.delete(ownerHash);
|
|
22970
|
+
}
|
|
22971
|
+
if (joinWasEmitted) {
|
|
22972
|
+
this._replicatorJoinEmitted.add(ownerHash);
|
|
22973
|
+
} else {
|
|
22974
|
+
this._replicatorJoinEmitted.delete(ownerHash);
|
|
22975
|
+
}
|
|
22976
|
+
},
|
|
22425
22977
|
};
|
|
22426
22978
|
}
|
|
22427
22979
|
|
|
22428
22980
|
const uniqueRanges = [
|
|
22429
22981
|
...new Map(ranges.map((range) => [range.idString, range])).values(),
|
|
22430
22982
|
];
|
|
22431
|
-
const wasReplicator = this.uniqueReplicators.has(ownerHash);
|
|
22432
|
-
const joinWasEmitted = this._replicatorJoinEmitted.has(ownerHash);
|
|
22433
22983
|
type Snapshot = {
|
|
22434
22984
|
range: ReplicationRangeIndexable<R>;
|
|
22435
22985
|
pending?: PendingMaturityRecord<R>;
|
|
@@ -22663,10 +23213,80 @@ export class SharedLog<
|
|
|
22663
23213
|
);
|
|
22664
23214
|
this.poisonReplicationOwnership(outcomeError);
|
|
22665
23215
|
}
|
|
23216
|
+
let rollbackStarted = false;
|
|
23217
|
+
const rollback = async () => {
|
|
23218
|
+
if (rollbackStarted) {
|
|
23219
|
+
return;
|
|
23220
|
+
}
|
|
23221
|
+
rollbackStarted = true;
|
|
23222
|
+
const rollbackErrors: unknown[] = [];
|
|
23223
|
+
const removedIds = new Set(removed.map((range) => range.idString));
|
|
23224
|
+
for (const snapshot of snapshots) {
|
|
23225
|
+
if (!removedIds.has(snapshot.range.idString)) {
|
|
23226
|
+
continue;
|
|
23227
|
+
}
|
|
23228
|
+
try {
|
|
23229
|
+
await this.replicationIndex.put(snapshot.range);
|
|
23230
|
+
this.putNativeReplicationRange(snapshot.range);
|
|
23231
|
+
} catch (error) {
|
|
23232
|
+
rollbackErrors.push(error);
|
|
23233
|
+
}
|
|
23234
|
+
}
|
|
23235
|
+
for (const snapshot of snapshots) {
|
|
23236
|
+
if (
|
|
23237
|
+
!removedIds.has(snapshot.range.idString) ||
|
|
23238
|
+
!snapshot.pending ||
|
|
23239
|
+
this.pendingMaturity.get(ownerHash)?.has(snapshot.range.idString)
|
|
23240
|
+
) {
|
|
23241
|
+
continue;
|
|
23242
|
+
}
|
|
23243
|
+
try {
|
|
23244
|
+
this.schedulePendingMaturity(
|
|
23245
|
+
snapshot.pending.range,
|
|
23246
|
+
snapshot.pending.from,
|
|
23247
|
+
{
|
|
23248
|
+
rebalance: snapshot.pending.rebalance,
|
|
23249
|
+
waitMs: Math.max(0, snapshot.pending.expiresAt - Date.now()),
|
|
23250
|
+
},
|
|
23251
|
+
snapshot.pending.ownershipLifecycleController,
|
|
23252
|
+
);
|
|
23253
|
+
} catch (error) {
|
|
23254
|
+
rollbackErrors.push(error);
|
|
23255
|
+
}
|
|
23256
|
+
}
|
|
23257
|
+
try {
|
|
23258
|
+
if (wasReplicator) {
|
|
23259
|
+
this.uniqueReplicators.add(ownerHash);
|
|
23260
|
+
} else {
|
|
23261
|
+
this.uniqueReplicators.delete(ownerHash);
|
|
23262
|
+
}
|
|
23263
|
+
if (joinWasEmitted) {
|
|
23264
|
+
this._replicatorJoinEmitted.add(ownerHash);
|
|
23265
|
+
} else {
|
|
23266
|
+
this._replicatorJoinEmitted.delete(ownerHash);
|
|
23267
|
+
}
|
|
23268
|
+
} catch (error) {
|
|
23269
|
+
rollbackErrors.push(error);
|
|
23270
|
+
}
|
|
23271
|
+
try {
|
|
23272
|
+
await this.updateOldestTimestampFromIndex();
|
|
23273
|
+
} catch (error) {
|
|
23274
|
+
rollbackErrors.push(error);
|
|
23275
|
+
}
|
|
23276
|
+
if (rollbackErrors.length > 0) {
|
|
23277
|
+
const failure = new AggregateError(
|
|
23278
|
+
rollbackErrors,
|
|
23279
|
+
"Replication-range deletion rollback failed",
|
|
23280
|
+
);
|
|
23281
|
+
this.poisonReplicationOwnership(failure);
|
|
23282
|
+
throw failure;
|
|
23283
|
+
}
|
|
23284
|
+
};
|
|
22666
23285
|
return {
|
|
22667
23286
|
removed,
|
|
22668
23287
|
retained,
|
|
22669
23288
|
ownerHasRanges,
|
|
23289
|
+
rollback,
|
|
22670
23290
|
error: outcomeError,
|
|
22671
23291
|
};
|
|
22672
23292
|
}
|
|
@@ -22682,7 +23302,8 @@ export class SharedLog<
|
|
|
22682
23302
|
|
|
22683
23303
|
private scheduleReplicationInfoRequests(
|
|
22684
23304
|
peer: PublicSignKey,
|
|
22685
|
-
replicationLifecycleController = this._instanceLifecycle
|
|
23305
|
+
replicationLifecycleController = this._instanceLifecycle
|
|
23306
|
+
?.membershipLifecycleController,
|
|
22686
23307
|
) {
|
|
22687
23308
|
if (
|
|
22688
23309
|
!replicationLifecycleController ||
|
|
@@ -22764,7 +23385,8 @@ export class SharedLog<
|
|
|
22764
23385
|
if (!topics.includes(this.topic)) {
|
|
22765
23386
|
return;
|
|
22766
23387
|
}
|
|
22767
|
-
const replicationLifecycleController =
|
|
23388
|
+
const replicationLifecycleController =
|
|
23389
|
+
this._instanceLifecycle?.membershipLifecycleController;
|
|
22768
23390
|
if (
|
|
22769
23391
|
!replicationLifecycleController ||
|
|
22770
23392
|
!this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
@@ -22775,10 +23397,7 @@ export class SharedLog<
|
|
|
22775
23397
|
const peerHash = publicKey.hashcode();
|
|
22776
23398
|
const expectedSubscriptionEpoch =
|
|
22777
23399
|
subscriptionEpoch ??
|
|
22778
|
-
this._peerSessions.rotate(
|
|
22779
|
-
peerHash,
|
|
22780
|
-
subscribed ? "opening" : "departing",
|
|
22781
|
-
);
|
|
23400
|
+
this._peerSessions.rotate(peerHash, subscribed ? "opening" : "departing");
|
|
22782
23401
|
const ownsSubscriptionEpoch = () =>
|
|
22783
23402
|
this._peerSessions.isCurrent(peerHash, expectedSubscriptionEpoch);
|
|
22784
23403
|
if (!ownsSubscriptionEpoch()) {
|
|
@@ -22787,6 +23406,7 @@ export class SharedLog<
|
|
|
22787
23406
|
// A destination stream is scoped to exactly one topic-subscription
|
|
22788
23407
|
// session. Abort the predecessor synchronously before either barrier can
|
|
22789
23408
|
// yield; a late queue completion must never enter the new session.
|
|
23409
|
+
this._v2Receive.clearPeer(peerHash);
|
|
22790
23410
|
this._v2Send.clearPeer(peerHash);
|
|
22791
23411
|
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
22792
23412
|
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
@@ -22923,33 +23543,30 @@ export class SharedLog<
|
|
|
22923
23543
|
this._liveness._replicatorLivenessFailures.delete(peerHash);
|
|
22924
23544
|
this._liveness.markReplicatorActivity(peerHash);
|
|
22925
23545
|
this._peerSessions.markOpen(peerHash, expectedSubscriptionEpoch);
|
|
23546
|
+
this.promoteReplicationInfoV2ReceiveCapability(
|
|
23547
|
+
publicKey,
|
|
23548
|
+
expectedSubscriptionEpoch,
|
|
23549
|
+
);
|
|
22926
23550
|
|
|
22927
|
-
// Decode
|
|
22928
|
-
//
|
|
22929
|
-
//
|
|
22930
|
-
//
|
|
22931
|
-
|
|
22932
|
-
|
|
22933
|
-
|
|
22934
|
-
|
|
22935
|
-
|
|
22936
|
-
:
|
|
22937
|
-
|
|
22938
|
-
|
|
22939
|
-
|
|
22940
|
-
capabilities: receiveCapabilities,
|
|
22941
|
-
}),
|
|
22942
|
-
{
|
|
22943
|
-
mode: new SilentDelivery({ redundancy: 1, to: [publicKey] }),
|
|
22944
|
-
signal: replicationLifecycleController.signal,
|
|
22945
|
-
},
|
|
22946
|
-
)
|
|
22947
|
-
.catch((error) =>
|
|
23551
|
+
// Decode, sender and authenticated apply readiness are separate bits. Start
|
|
23552
|
+
// the ACKed receiver-led negotiation before the legacy snapshot, but do not
|
|
23553
|
+
// hold mixed-version replication discovery behind that round trip. Attach
|
|
23554
|
+
// the rejection handler now because a lifecycle fence below may return before
|
|
23555
|
+
// awaiting the advert. RequestV2 is armed only after both sends settle.
|
|
23556
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
23557
|
+
const localCapabilityAdvertisement =
|
|
23558
|
+
this.advertiseReplicationInfoV2ReceiveCapability({
|
|
23559
|
+
target: publicKey,
|
|
23560
|
+
peerSession: expectedSubscriptionEpoch,
|
|
23561
|
+
receiveEpoch,
|
|
23562
|
+
signal: replicationLifecycleController.signal,
|
|
23563
|
+
}).catch((error) => {
|
|
22948
23564
|
this.handleReplicationLifecycleSendError(
|
|
22949
23565
|
error,
|
|
22950
23566
|
replicationLifecycleController,
|
|
22951
|
-
)
|
|
22952
|
-
|
|
23567
|
+
);
|
|
23568
|
+
return undefined;
|
|
23569
|
+
});
|
|
22953
23570
|
|
|
22954
23571
|
let replicationSegments: ReplicationRangeIndexable<R>[];
|
|
22955
23572
|
try {
|
|
@@ -23019,9 +23636,9 @@ export class SharedLog<
|
|
|
23019
23636
|
}
|
|
23020
23637
|
}
|
|
23021
23638
|
|
|
23022
|
-
//
|
|
23023
|
-
// timing-sensitive delivery/order
|
|
23024
|
-
//
|
|
23639
|
+
// Keep legacy request-based discovery independent of the capability ACK too.
|
|
23640
|
+
// This makes mixed-version joins resilient to timing-sensitive delivery/order
|
|
23641
|
+
// issues where we may miss the remote peer's initial announcement.
|
|
23025
23642
|
if (
|
|
23026
23643
|
this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
23027
23644
|
ownsSubscriptionEpoch()
|
|
@@ -23031,6 +23648,16 @@ export class SharedLog<
|
|
|
23031
23648
|
replicationLifecycleController,
|
|
23032
23649
|
);
|
|
23033
23650
|
}
|
|
23651
|
+
|
|
23652
|
+
const localCapabilityReady = await localCapabilityAdvertisement;
|
|
23653
|
+
if (localCapabilityReady) {
|
|
23654
|
+
this._v2Receive.markLocalCapabilityReady({
|
|
23655
|
+
peerHash,
|
|
23656
|
+
peerSession: expectedSubscriptionEpoch,
|
|
23657
|
+
receiveEpoch,
|
|
23658
|
+
...localCapabilityReady,
|
|
23659
|
+
});
|
|
23660
|
+
}
|
|
23034
23661
|
}
|
|
23035
23662
|
|
|
23036
23663
|
private getClampedReplicas(customValue?: MinReplicas) {
|
|
@@ -24153,7 +24780,10 @@ export class SharedLog<
|
|
|
24153
24780
|
if (oldPeersSet) {
|
|
24154
24781
|
for (const oldPeer of oldPeersSet) {
|
|
24155
24782
|
if (!currentPeers.has(oldPeer)) {
|
|
24156
|
-
this._checkedPrune.removeRequestSent(
|
|
24783
|
+
this._checkedPrune.removeRequestSent(
|
|
24784
|
+
entryReplicated.hash,
|
|
24785
|
+
oldPeer,
|
|
24786
|
+
);
|
|
24157
24787
|
}
|
|
24158
24788
|
}
|
|
24159
24789
|
}
|
|
@@ -24242,7 +24872,10 @@ export class SharedLog<
|
|
|
24242
24872
|
if (oldPeersSet) {
|
|
24243
24873
|
for (const oldPeer of oldPeersSet) {
|
|
24244
24874
|
if (!currentPeers.has(oldPeer)) {
|
|
24245
|
-
this._checkedPrune.removeRequestSent(
|
|
24875
|
+
this._checkedPrune.removeRequestSent(
|
|
24876
|
+
entryReplicated.hash,
|
|
24877
|
+
oldPeer,
|
|
24878
|
+
);
|
|
24246
24879
|
}
|
|
24247
24880
|
}
|
|
24248
24881
|
}
|
|
@@ -24422,10 +25055,7 @@ export class SharedLog<
|
|
|
24422
25055
|
}
|
|
24423
25056
|
|
|
24424
25057
|
const fromHash = evt.detail.from.hashcode();
|
|
24425
|
-
const subscriptionEpoch = this._peerSessions.rotate(
|
|
24426
|
-
fromHash,
|
|
24427
|
-
"departing",
|
|
24428
|
-
);
|
|
25058
|
+
const subscriptionEpoch = this._peerSessions.rotate(fromHash, "departing");
|
|
24429
25059
|
this._peerSessions.blockReplicationInfo(fromHash);
|
|
24430
25060
|
this._recentRepairDispatch.delete(fromHash);
|
|
24431
25061
|
|