@peerbit/shared-log 15.0.0 → 16.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +7 -12
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +66 -580
- package/dist/src/index.js.map +1 -1
- package/dist/src/replication-announcement.d.ts +3 -106
- package/dist/src/replication-announcement.d.ts.map +1 -1
- package/dist/src/replication-announcement.js +0 -521
- package/dist/src/replication-announcement.js.map +1 -1
- package/dist/src/replication-info-v2-receive.d.ts +10 -41
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -1
- package/dist/src/replication-info-v2-receive.js +11 -192
- package/dist/src/replication-info-v2-receive.js.map +1 -1
- package/dist/src/replication-info-v2-send.d.ts +0 -1
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +0 -6
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/dist/src/replication.d.ts +0 -1
- package/dist/src/replication.d.ts.map +1 -1
- package/dist/src/replication.js +2 -17
- package/dist/src/replication.js.map +1 -1
- package/dist/src/sync/factory.d.ts +0 -1
- package/dist/src/sync/factory.d.ts.map +1 -1
- package/dist/src/sync/factory.js +20 -36
- package/dist/src/sync/factory.js.map +1 -1
- package/package.json +10 -10
- package/src/index.ts +87 -794
- package/src/replication-announcement.ts +8 -739
- package/src/replication-info-v2-receive.ts +17 -263
- package/src/replication-info-v2-send.ts +0 -7
- package/src/replication.ts +1 -20
- package/src/sync/factory.ts +24 -38
package/src/index.ts
CHANGED
|
@@ -231,10 +231,7 @@ import {
|
|
|
231
231
|
shouldAssigneToRangeBoundary as shouldAssignToRangeBoundary,
|
|
232
232
|
toRebalance,
|
|
233
233
|
} from "./ranges.js";
|
|
234
|
-
import {
|
|
235
|
-
ReplicationAnnouncementCoordinator,
|
|
236
|
-
isTransientReplicationAnnouncementError,
|
|
237
|
-
} from "./replication-announcement.js";
|
|
234
|
+
import { ReplicationAnnouncementCoordinator } from "./replication-announcement.js";
|
|
238
235
|
import {
|
|
239
236
|
type ReplicationDomainHash,
|
|
240
237
|
createReplicationDomainHash,
|
|
@@ -277,7 +274,6 @@ import {
|
|
|
277
274
|
maxReplicas,
|
|
278
275
|
} from "./replication.js";
|
|
279
276
|
import { ReplicatorLivenessMonitor } from "./replicator-liveness.js";
|
|
280
|
-
import { Observer, Replicator } from "./role.js";
|
|
281
277
|
import { createSyncronizer } from "./sync/factory.js";
|
|
282
278
|
import type {
|
|
283
279
|
SharedLogNativeWireSync,
|
|
@@ -2078,34 +2074,26 @@ export class SharedLog<
|
|
|
2078
2074
|
// public key hash to range id to range
|
|
2079
2075
|
pendingMaturity!: Map<string, Map<string, PendingMaturityRecord<R>>>; // map of peerId to timeout
|
|
2080
2076
|
|
|
2081
|
-
//
|
|
2082
|
-
// FENCING role
|
|
2083
|
-
//
|
|
2084
|
-
// the
|
|
2085
|
-
//
|
|
2086
|
-
//
|
|
2087
|
-
// NOT subsumed: within one (lifecycle, session, epoch, unblocked) regime
|
|
2088
|
-
// the epoch token is constant across every message from the peer, so only
|
|
2089
|
-
// the two apply-lane timestamp comparisons can drop an older reset
|
|
2090
|
-
// delivered after a newer add (unordered pubsub / retransmits) — an
|
|
2091
|
-
// identity token carries no order. Deletion is blocked until
|
|
2092
|
-
// replication-info messages carry sender-authoritative sequence numbers
|
|
2093
|
-
// (stage-5 schema change); the `receive admission replication-info
|
|
2094
|
-
// ordering watermark` pins fail if the read sites are removed before then.
|
|
2095
|
-
private latestReplicationInfoMessage!: Map<string, bigint>;
|
|
2077
|
+
// The legacy replication-info ordering watermark (fence B8) is deleted:
|
|
2078
|
+
// its FENCING role was subsumed by the per-peer receive epoch, blocked set
|
|
2079
|
+
// and session identity, and its intra-epoch ORDERING role existed only for
|
|
2080
|
+
// the legacy apply lanes. The V2 lane orders by sender-authoritative
|
|
2081
|
+
// sequence numbers, and legacy frames are dropped unconditionally at the
|
|
2082
|
+
// B1 gate before any side effect.
|
|
2096
2083
|
// The replication-info blocked set (fence B5) lives on the peer-session
|
|
2097
2084
|
// registry: unsubscribed peers whose replication-info is ignored until a
|
|
2098
2085
|
// reconnect barrier commits. See PeerSessionRegistry._replicationInfoBlockedPeers.
|
|
2086
|
+
// V2 recovery scheduler state, one row per open peer session (B12: the
|
|
2087
|
+
// legacy request scheduler that shared this map is deleted).
|
|
2099
2088
|
private _replicationInfoRequestByPeer!: Map<
|
|
2100
2089
|
string,
|
|
2101
2090
|
{
|
|
2102
|
-
//
|
|
2103
|
-
//
|
|
2104
|
-
// for the next unpark delay, reset on any applied V2 progress.
|
|
2091
|
+
// Consecutive fruitless unparks — the escalation exponent for the
|
|
2092
|
+
// next unpark delay, reset on any applied V2 progress.
|
|
2105
2093
|
attempts: number;
|
|
2106
2094
|
timer?: ReturnType<typeof setTimeout>;
|
|
2107
|
-
peerSession
|
|
2108
|
-
//
|
|
2095
|
+
peerSession: PeerSession;
|
|
2096
|
+
// When the current park was first observed.
|
|
2109
2097
|
parkedSinceMs?: number;
|
|
2110
2098
|
}
|
|
2111
2099
|
>;
|
|
@@ -2266,7 +2254,6 @@ export class SharedLog<
|
|
|
2266
2254
|
for (const hash of this._checkedPrune?.retries.keys() ?? []) {
|
|
2267
2255
|
this._checkedPrune.clearRetry(hash);
|
|
2268
2256
|
}
|
|
2269
|
-
this._announcements.cancelCurrentReplicationStateAnnouncementRetry();
|
|
2270
2257
|
this.joinWarmup.cancelAllJoinWarmupTargets();
|
|
2271
2258
|
for (const timer of this._repairRetryTimers) {
|
|
2272
2259
|
clearTimeout(timer);
|
|
@@ -3268,7 +3255,7 @@ export class SharedLog<
|
|
|
3268
3255
|
private rebalanceParticipationDebounced:
|
|
3269
3256
|
| ReturnType<typeof debounceFixedInterval>
|
|
3270
3257
|
| undefined;
|
|
3271
|
-
private _announcements!: ReplicationAnnouncementCoordinator
|
|
3258
|
+
private _announcements!: ReplicationAnnouncementCoordinator;
|
|
3272
3259
|
private _v2Receive!: ReplicationInfoV2ReceiveCoordinator;
|
|
3273
3260
|
private _v2Send!: ReplicationInfoV2SendCoordinator<R>;
|
|
3274
3261
|
|
|
@@ -3396,37 +3383,13 @@ export class SharedLog<
|
|
|
3396
3383
|
});
|
|
3397
3384
|
}
|
|
3398
3385
|
|
|
3399
|
-
private createReplicationAnnouncementCoordinator(): ReplicationAnnouncementCoordinator
|
|
3400
|
-
return new ReplicationAnnouncementCoordinator
|
|
3401
|
-
// Route re-entrant queueing through the owner so coordinator spies keep
|
|
3402
|
-
// observing it (the poison guard assertions in events.spec.ts depend on
|
|
3403
|
-
// this).
|
|
3404
|
-
queueCurrentReplicationStateAnnouncementRepair: () =>
|
|
3405
|
-
this._announcements.queueCurrentReplicationStateAnnouncementRepair(),
|
|
3406
|
-
queueCurrentReplicationStateAnnouncementRetry: (error: unknown) =>
|
|
3407
|
-
this._announcements.queueCurrentReplicationStateAnnouncementRetry(
|
|
3408
|
-
error,
|
|
3409
|
-
),
|
|
3386
|
+
private createReplicationAnnouncementCoordinator(): ReplicationAnnouncementCoordinator {
|
|
3387
|
+
return new ReplicationAnnouncementCoordinator({
|
|
3410
3388
|
enqueueReplicationInfoV2: (message) => this._v2Send.enqueue(message),
|
|
3411
|
-
isLegacyReplicationInfoEnabled: () => this.legacyReplicationInfoEnabled,
|
|
3412
|
-
isClosed: () => this.closed,
|
|
3413
|
-
getCloseSignal: () => this._closeController.signal,
|
|
3414
|
-
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
3415
|
-
validatePersistedReplicationRangeSnapshot: (ranges) =>
|
|
3416
|
-
this.validatePersistedReplicationRangeSnapshot(ranges),
|
|
3417
|
-
getSubscribers: () =>
|
|
3418
|
-
this.node.services.pubsub.getSubscribers(this.topic),
|
|
3419
|
-
getSelfHash: () => this.node.identity.publicKey.hashcode(),
|
|
3420
|
-
isBlockedPeer: (hash) =>
|
|
3421
|
-
this._peerSessions.isReplicationInfoBlocked(hash),
|
|
3422
|
-
getRpc: () => this.rpc,
|
|
3423
3389
|
captureReplicationOwnershipLifecycle: () =>
|
|
3424
3390
|
this.captureReplicationOwnershipLifecycle(),
|
|
3425
3391
|
throwIfReplicationOwnershipLifecycleInactive: (controller) =>
|
|
3426
3392
|
this.throwIfReplicationOwnershipLifecycleInactive(controller),
|
|
3427
|
-
isAdaptiveReplicating: () => this._isAdaptiveReplicating,
|
|
3428
|
-
callRebalanceParticipationDebounced: () =>
|
|
3429
|
-
this.rebalanceParticipationDebounced?.call(),
|
|
3430
3393
|
});
|
|
3431
3394
|
}
|
|
3432
3395
|
|
|
@@ -3653,7 +3616,6 @@ export class SharedLog<
|
|
|
3653
3616
|
this._admittedPruneRemoves = new Set();
|
|
3654
3617
|
this._pendingIHave = new Map();
|
|
3655
3618
|
this._pendingIHaveCallbacks = new Set();
|
|
3656
|
-
this.latestReplicationInfoMessage = new Map();
|
|
3657
3619
|
this._replicationInfoRequestByPeer = new Map();
|
|
3658
3620
|
this._subscriberSnapshotRequestsByPeer = new Map();
|
|
3659
3621
|
this._replicationInfoApplyQueueByPeer = new Map();
|
|
@@ -3764,29 +3726,10 @@ export class SharedLog<
|
|
|
3764
3726
|
this._nativeStrictDurableTransactionsClosing ??= false;
|
|
3765
3727
|
}
|
|
3766
3728
|
|
|
3767
|
-
get compatibility(): number | undefined {
|
|
3768
|
-
// B12: the open option was removed and any defined value rejects at
|
|
3769
|
-
// open(); this is permanently undefined and dies with the residual
|
|
3770
|
-
// gates in a later cleanup stage.
|
|
3771
|
-
return (this._logProperties as any)?.compatibility;
|
|
3772
|
-
}
|
|
3773
|
-
|
|
3774
|
-
/**
|
|
3775
|
-
* Legacy replication-info is an explicit compatibility fallback. Current
|
|
3776
|
-
* logs never infer or re-enable it from a remote peer's capabilities.
|
|
3777
|
-
*/
|
|
3778
|
-
private get legacyReplicationInfoEnabled(): boolean {
|
|
3779
|
-
return this.compatibility !== undefined && this.compatibility < 10;
|
|
3780
|
-
}
|
|
3781
|
-
|
|
3782
3729
|
get isAdaptiveReplicating() {
|
|
3783
3730
|
return this._isAdaptiveReplicating;
|
|
3784
3731
|
}
|
|
3785
3732
|
|
|
3786
|
-
private get v8Behaviour() {
|
|
3787
|
-
return (this.compatibility ?? Number.MAX_VALUE) < 9;
|
|
3788
|
-
}
|
|
3789
|
-
|
|
3790
3733
|
private getFanoutChannelOptions(
|
|
3791
3734
|
options?: SharedLogFanoutOptions,
|
|
3792
3735
|
): Omit<FanoutTreeChannelOptions, "role"> {
|
|
@@ -4691,8 +4634,7 @@ export class SharedLog<
|
|
|
4691
4634
|
(leaders.size === 0 || (leaders.size === 1 && leaders.has(selfHash)))
|
|
4692
4635
|
) {
|
|
4693
4636
|
const allowSubscriberFallback =
|
|
4694
|
-
this.syncronizer instanceof SimpleSyncronizer
|
|
4695
|
-
(this.compatibility ?? Number.MAX_VALUE) < 10;
|
|
4637
|
+
this.syncronizer instanceof SimpleSyncronizer;
|
|
4696
4638
|
if (!allowSubscriberFallback) {
|
|
4697
4639
|
return;
|
|
4698
4640
|
}
|
|
@@ -4841,8 +4783,7 @@ export class SharedLog<
|
|
|
4841
4783
|
const set = new Set(leaders.keys());
|
|
4842
4784
|
let hasRemotePeers = set.has(selfHash) ? set.size > 1 : set.size > 0;
|
|
4843
4785
|
const allowSubscriberFallback =
|
|
4844
|
-
this.syncronizer instanceof SimpleSyncronizer
|
|
4845
|
-
(this.compatibility ?? Number.MAX_VALUE) < 10;
|
|
4786
|
+
this.syncronizer instanceof SimpleSyncronizer;
|
|
4846
4787
|
if (!hasRemotePeers && allowSubscriberFallback) {
|
|
4847
4788
|
try {
|
|
4848
4789
|
const subscribers = await this._getTopicSubscribers(this.topic);
|
|
@@ -5305,28 +5246,6 @@ export class SharedLog<
|
|
|
5305
5246
|
};
|
|
5306
5247
|
}
|
|
5307
5248
|
|
|
5308
|
-
// @deprecated
|
|
5309
|
-
private getRoleFromReplicationSegments(
|
|
5310
|
-
segments: ReplicationRangeIndexable<R>[],
|
|
5311
|
-
) {
|
|
5312
|
-
if (segments.length > 1) {
|
|
5313
|
-
throw new Error(
|
|
5314
|
-
"More than one replication segment found. Can only use one segment for compatbility with v8",
|
|
5315
|
-
);
|
|
5316
|
-
}
|
|
5317
|
-
|
|
5318
|
-
if (segments.length > 0) {
|
|
5319
|
-
const segment = segments[0].toReplicationRange();
|
|
5320
|
-
return new Replicator({
|
|
5321
|
-
factor: (segment.factor as number) / MAX_U32,
|
|
5322
|
-
offset: (segment.offset as number) / MAX_U32,
|
|
5323
|
-
});
|
|
5324
|
-
}
|
|
5325
|
-
|
|
5326
|
-
// TODO this is not accurate but might be good enough
|
|
5327
|
-
return new Observer();
|
|
5328
|
-
}
|
|
5329
|
-
|
|
5330
5249
|
private isTerminating() {
|
|
5331
5250
|
return (
|
|
5332
5251
|
this.acceptsParentAttachments === false ||
|
|
@@ -5604,12 +5523,7 @@ export class SharedLog<
|
|
|
5604
5523
|
}
|
|
5605
5524
|
|
|
5606
5525
|
private onRebalanceParticipationError(error: Error): void {
|
|
5607
|
-
if (
|
|
5608
|
-
this.closed ||
|
|
5609
|
-
isNotStartedError(error) ||
|
|
5610
|
-
(isTransientReplicationAnnouncementError(error) &&
|
|
5611
|
-
this._announcements._replicationAnnouncementRetryPending)
|
|
5612
|
-
) {
|
|
5526
|
+
if (this.closed || isNotStartedError(error)) {
|
|
5613
5527
|
return;
|
|
5614
5528
|
}
|
|
5615
5529
|
|
|
@@ -14247,12 +14161,7 @@ export class SharedLog<
|
|
|
14247
14161
|
|
|
14248
14162
|
this.domain = options?.domain
|
|
14249
14163
|
? (options.domain(this) as unknown as D)
|
|
14250
|
-
: (createReplicationDomainHash(
|
|
14251
|
-
(options as any)?.compatibility !== undefined &&
|
|
14252
|
-
(options as any).compatibility < 10
|
|
14253
|
-
? "u32"
|
|
14254
|
-
: "u64",
|
|
14255
|
-
)(this) as unknown as D);
|
|
14164
|
+
: (createReplicationDomainHash("u64")(this) as unknown as D);
|
|
14256
14165
|
this.indexableDomain = createIndexableDomainFromResolution(
|
|
14257
14166
|
this.domain.resolution,
|
|
14258
14167
|
);
|
|
@@ -14262,7 +14171,6 @@ export class SharedLog<
|
|
|
14262
14171
|
this._admittedPruneRemoves = new Set();
|
|
14263
14172
|
this._pendingIHave = new Map();
|
|
14264
14173
|
this._pendingIHaveCallbacks = new Set();
|
|
14265
|
-
this.latestReplicationInfoMessage = new Map();
|
|
14266
14174
|
this._replicationInfoRequestByPeer = new Map();
|
|
14267
14175
|
this._subscriberSnapshotRequestsByPeer = new Map();
|
|
14268
14176
|
// Terminal close/drop drains the previous lifecycle before another open can
|
|
@@ -14335,7 +14243,6 @@ export class SharedLog<
|
|
|
14335
14243
|
this._liveness.resetForOpen();
|
|
14336
14244
|
this._lastLocalAppendAt = 0;
|
|
14337
14245
|
this._announcements ??= this.createReplicationAnnouncementCoordinator();
|
|
14338
|
-
this._announcements.resetForOpen();
|
|
14339
14246
|
this._v2Receive ??= this.createReplicationInfoV2ReceiveCoordinator();
|
|
14340
14247
|
this._v2Receive.resetForOpen();
|
|
14341
14248
|
this._v2Send ??= this.createReplicationInfoV2SendCoordinator();
|
|
@@ -14398,12 +14305,6 @@ export class SharedLog<
|
|
|
14398
14305
|
}
|
|
14399
14306
|
|
|
14400
14307
|
this._closeController = new AbortController();
|
|
14401
|
-
if (this.legacyReplicationInfoEnabled) {
|
|
14402
|
-
this._announcements.setupReplicationAnnouncementRetryFunction();
|
|
14403
|
-
this._announcements.setupReplicationAnnouncementRepairFunction();
|
|
14404
|
-
} else {
|
|
14405
|
-
this._announcements.cancelCurrentReplicationStateAnnouncementRetry();
|
|
14406
|
-
}
|
|
14407
14308
|
this._closeController.signal.addEventListener("abort", () => {
|
|
14408
14309
|
for (const [_peer, state] of this._replicationInfoRequestByPeer) {
|
|
14409
14310
|
if (state.timer) clearTimeout(state.timer);
|
|
@@ -14859,7 +14760,6 @@ export class SharedLog<
|
|
|
14859
14760
|
sendOptions?: { priority?: number; signal?: AbortSignal },
|
|
14860
14761
|
) => this.trySendFusedRawExchangeHeads(hashes, to, sendOptions),
|
|
14861
14762
|
warn,
|
|
14862
|
-
compatibility: (this._logProperties as any)?.compatibility,
|
|
14863
14763
|
resolution: this.domain.resolution,
|
|
14864
14764
|
sync: options?.sync,
|
|
14865
14765
|
syncronizer: options?.syncronizer,
|
|
@@ -15863,7 +15763,7 @@ export class SharedLog<
|
|
|
15863
15763
|
const peerSession = this._peerSessions.current(peerHash);
|
|
15864
15764
|
const preserveV2Session =
|
|
15865
15765
|
peerSession?.phase === "open" && peerSession.isActive();
|
|
15866
|
-
if (
|
|
15766
|
+
if (!preserveV2Session) {
|
|
15867
15767
|
this.cancelReplicationInfoRequests(peerHash);
|
|
15868
15768
|
}
|
|
15869
15769
|
this._liveness._replicatorLivenessFailures.delete(peerHash);
|
|
@@ -15895,11 +15795,8 @@ export class SharedLog<
|
|
|
15895
15795
|
|
|
15896
15796
|
private advanceReplicationInfoRecoveryEpoch(peerHash: string) {
|
|
15897
15797
|
// Handlers admitted before a successful peer removal must not restore state
|
|
15898
|
-
// when they eventually reach the apply lane.
|
|
15899
|
-
// ordering watermark with the local epoch so a later arrival can be
|
|
15900
|
-
// accepted without comparing its clock to this receiver's clock.
|
|
15798
|
+
// when they eventually reach the apply lane.
|
|
15901
15799
|
const receiveEpoch = this._peerSessions.advanceReceiveEpoch(peerHash);
|
|
15902
|
-
this.latestReplicationInfoMessage.delete(peerHash);
|
|
15903
15800
|
const peerSession = this._peerSessions.current(peerHash);
|
|
15904
15801
|
if (peerSession?.phase === "open") {
|
|
15905
15802
|
this._v2Receive.advanceRecovery({
|
|
@@ -16788,15 +16685,6 @@ export class SharedLog<
|
|
|
16788
16685
|
firstError ??= error;
|
|
16789
16686
|
}
|
|
16790
16687
|
};
|
|
16791
|
-
// A borsh-deserialized instance that is closed before ever being opened
|
|
16792
|
-
// has no coordinators (they are created in open()); the old inline code
|
|
16793
|
-
// only touched plain fields here and never threw.
|
|
16794
|
-
captureSync(() =>
|
|
16795
|
-
this._announcements?.cancelCurrentReplicationStateAnnouncementRetry(),
|
|
16796
|
-
);
|
|
16797
|
-
if (this._announcements) {
|
|
16798
|
-
this._announcements.replicationAnnouncementRetryDebounced = undefined;
|
|
16799
|
-
}
|
|
16800
16688
|
captureSync(() => {
|
|
16801
16689
|
if (this._wireSyncSession) {
|
|
16802
16690
|
this._wireSyncSession.unregisterTopic(this.topic);
|
|
@@ -16890,7 +16778,6 @@ export class SharedLog<
|
|
|
16890
16778
|
captureSync(() => {
|
|
16891
16779
|
this._pendingIHave?.clear();
|
|
16892
16780
|
this._pendingIHaveCallbacks?.clear();
|
|
16893
|
-
this.latestReplicationInfoMessage?.clear();
|
|
16894
16781
|
this._peerSessions?.clearReceiveEpochsForClose();
|
|
16895
16782
|
this._peerSessions?.clearCleanupGatesForClose();
|
|
16896
16783
|
this._activeReceiveHandlersByPeer?.clear();
|
|
@@ -16991,7 +16878,6 @@ export class SharedLog<
|
|
|
16991
16878
|
await pruneRemoveTerminalFence.drained;
|
|
16992
16879
|
await this.drainPendingIHaveCallbacks();
|
|
16993
16880
|
this.ensureNativeDurabilityRuntimeState();
|
|
16994
|
-
this._announcements.cancelCurrentReplicationStateAnnouncementRetry();
|
|
16995
16881
|
} catch (error) {
|
|
16996
16882
|
// The terminal preamble has already disabled parent attachments and the
|
|
16997
16883
|
// network lifecycle. Keep mutation admission fenced for an exact retry.
|
|
@@ -17035,19 +16921,7 @@ export class SharedLog<
|
|
|
17035
16921
|
}
|
|
17036
16922
|
}, 2_000);
|
|
17037
16923
|
try {
|
|
17038
|
-
|
|
17039
|
-
const resets = [this._v2Send.sendTerminalReset(abort.signal)];
|
|
17040
|
-
if (this.legacyReplicationInfoEnabled) {
|
|
17041
|
-
resets.push(
|
|
17042
|
-
this.rpc
|
|
17043
|
-
.send(reset, {
|
|
17044
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
17045
|
-
signal: abort.signal,
|
|
17046
|
-
})
|
|
17047
|
-
.catch(() => {}),
|
|
17048
|
-
);
|
|
17049
|
-
}
|
|
17050
|
-
await Promise.all(resets);
|
|
16924
|
+
await this._v2Send.sendTerminalReset(abort.signal);
|
|
17051
16925
|
} finally {
|
|
17052
16926
|
clearTimeout(abortTimer);
|
|
17053
16927
|
}
|
|
@@ -17140,7 +17014,6 @@ export class SharedLog<
|
|
|
17140
17014
|
await replicationRangeTerminalFence.drained;
|
|
17141
17015
|
await pruneRemoveTerminalFence.drained;
|
|
17142
17016
|
await this.drainPendingIHaveCallbacks();
|
|
17143
|
-
this._announcements.cancelCurrentReplicationStateAnnouncementRetry();
|
|
17144
17017
|
} catch (error) {
|
|
17145
17018
|
// The terminal preamble is not safely reversible. Preserve the fence until
|
|
17146
17019
|
// a retry finishes cleanup.
|
|
@@ -17168,19 +17041,7 @@ export class SharedLog<
|
|
|
17168
17041
|
}
|
|
17169
17042
|
}, 2_000);
|
|
17170
17043
|
try {
|
|
17171
|
-
|
|
17172
|
-
const resets = [this._v2Send.sendTerminalReset(abort.signal)];
|
|
17173
|
-
if (this.legacyReplicationInfoEnabled) {
|
|
17174
|
-
resets.push(
|
|
17175
|
-
this.rpc
|
|
17176
|
-
.send(reset, {
|
|
17177
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
17178
|
-
signal: abort.signal,
|
|
17179
|
-
})
|
|
17180
|
-
.catch(() => {}),
|
|
17181
|
-
);
|
|
17182
|
-
}
|
|
17183
|
-
await Promise.all(resets);
|
|
17044
|
+
await this._v2Send.sendTerminalReset(abort.signal);
|
|
17184
17045
|
} finally {
|
|
17185
17046
|
clearTimeout(abortTimer);
|
|
17186
17047
|
}
|
|
@@ -17707,16 +17568,16 @@ export class SharedLog<
|
|
|
17707
17568
|
throw new Error("Missing from in update role message");
|
|
17708
17569
|
}
|
|
17709
17570
|
if (
|
|
17710
|
-
|
|
17711
|
-
|
|
17712
|
-
|
|
17713
|
-
|
|
17714
|
-
|
|
17715
|
-
msg instanceof StoppedReplicating)
|
|
17571
|
+
msg instanceof RequestReplicationInfoMessage ||
|
|
17572
|
+
msg instanceof ResponseRoleMessage ||
|
|
17573
|
+
msg instanceof AllReplicatingSegmentsMessage ||
|
|
17574
|
+
msg instanceof AddedReplicationSegmentMessage ||
|
|
17575
|
+
msg instanceof StoppedReplicating
|
|
17716
17576
|
) {
|
|
17717
|
-
// These variants remain registered decode tombstones, but
|
|
17718
|
-
//
|
|
17719
|
-
//
|
|
17577
|
+
// These variants remain registered decode tombstones, but logs fail
|
|
17578
|
+
// closed unconditionally before leases, synchronizer work, liveness,
|
|
17579
|
+
// watermarks or mutations (B12: the compatibility opens that once
|
|
17580
|
+
// admitted them reject at open()).
|
|
17720
17581
|
return;
|
|
17721
17582
|
}
|
|
17722
17583
|
// Snapshot receive ownership before any async handler gets a chance to
|
|
@@ -17757,22 +17618,14 @@ export class SharedLog<
|
|
|
17757
17618
|
this.captureReplicationOwnershipLifecycle();
|
|
17758
17619
|
const receiveReplicationInfoReceiveEpoch =
|
|
17759
17620
|
this._peerSessions.receiveEpoch(receiveFromHash);
|
|
17760
|
-
if (msg instanceof ResponseRoleMessage) {
|
|
17761
|
-
msg = msg.toReplicationInfoMessage(); // migration
|
|
17762
|
-
}
|
|
17763
17621
|
if (
|
|
17764
|
-
msg instanceof AllReplicatingSegmentsMessage ||
|
|
17765
|
-
msg instanceof AddedReplicationSegmentMessage ||
|
|
17766
17622
|
msg instanceof FullReplicationInfoV2Message ||
|
|
17767
17623
|
msg instanceof AddedReplicationInfoV2Message
|
|
17768
17624
|
) {
|
|
17769
17625
|
// Bound decoded untrusted vectors before per-peer/global mutation
|
|
17770
17626
|
// queues, trusted-replicator authorization, or liveness side effects.
|
|
17771
17627
|
this.validateReplicationRangeAnnouncement(msg.segments);
|
|
17772
|
-
} else if (
|
|
17773
|
-
msg instanceof StoppedReplicating ||
|
|
17774
|
-
msg instanceof StoppedReplicationInfoV2Message
|
|
17775
|
-
) {
|
|
17628
|
+
} else if (msg instanceof StoppedReplicationInfoV2Message) {
|
|
17776
17629
|
// Bound the raw decoded vector before deduplication can hide the
|
|
17777
17630
|
// allocation cost, and before liveness or apply-queue side effects.
|
|
17778
17631
|
this.validateStoppedReplicationAnnouncement(msg.segmentIds);
|
|
@@ -17780,10 +17633,7 @@ export class SharedLog<
|
|
|
17780
17633
|
if (
|
|
17781
17634
|
!context.from.equals(this.node.identity.publicKey) &&
|
|
17782
17635
|
!(msg instanceof RequestReplicationInfoV2Message) &&
|
|
17783
|
-
!isReplicationInfoV2Message(msg)
|
|
17784
|
-
!(msg instanceof AllReplicatingSegmentsMessage) &&
|
|
17785
|
-
!(msg instanceof AddedReplicationSegmentMessage) &&
|
|
17786
|
-
!(msg instanceof StoppedReplicating)
|
|
17636
|
+
!isReplicationInfoV2Message(msg)
|
|
17787
17637
|
) {
|
|
17788
17638
|
this._liveness.markReplicatorActivity(receiveFromHash);
|
|
17789
17639
|
}
|
|
@@ -19498,23 +19348,6 @@ export class SharedLog<
|
|
|
19498
19348
|
});
|
|
19499
19349
|
} else if (msg instanceof ReplicationPingMessage) {
|
|
19500
19350
|
// No-op: used as an ACKed unicast liveness probe.
|
|
19501
|
-
} else if (msg instanceof RequestReplicationInfoMessage) {
|
|
19502
|
-
await this.handleRequestReplicationInfo(
|
|
19503
|
-
msg,
|
|
19504
|
-
laneRequestContext,
|
|
19505
|
-
lane,
|
|
19506
|
-
);
|
|
19507
|
-
} else if (
|
|
19508
|
-
msg instanceof AllReplicatingSegmentsMessage ||
|
|
19509
|
-
msg instanceof AddedReplicationSegmentMessage
|
|
19510
|
-
) {
|
|
19511
|
-
await this.handleReplicationInfoAnnouncement(
|
|
19512
|
-
msg,
|
|
19513
|
-
laneRequestContext,
|
|
19514
|
-
lane,
|
|
19515
|
-
);
|
|
19516
|
-
} else if (msg instanceof StoppedReplicating) {
|
|
19517
|
-
await this.handleStoppedReplicating(msg, laneRequestContext, lane);
|
|
19518
19351
|
} else {
|
|
19519
19352
|
throw new Error("Unexpected message");
|
|
19520
19353
|
}
|
|
@@ -19743,113 +19576,6 @@ export class SharedLog<
|
|
|
19743
19576
|
}
|
|
19744
19577
|
}
|
|
19745
19578
|
|
|
19746
|
-
private async handleRequestReplicationInfo(
|
|
19747
|
-
_msg: RequestReplicationInfoMessage,
|
|
19748
|
-
context: ReceiveRequestContext,
|
|
19749
|
-
lane: ReceiveLaneContext,
|
|
19750
|
-
): Promise<void> {
|
|
19751
|
-
const receiveFromHash = lane.fromHash;
|
|
19752
|
-
const receiveSession = lane.session;
|
|
19753
|
-
const receiveReplicationLifecycleController = lane.lifecycleController;
|
|
19754
|
-
if (context.from.equals(this.node.identity.publicKey)) {
|
|
19755
|
-
return;
|
|
19756
|
-
}
|
|
19757
|
-
const replicationLifecycleController =
|
|
19758
|
-
receiveReplicationLifecycleController;
|
|
19759
|
-
if (
|
|
19760
|
-
!replicationLifecycleController ||
|
|
19761
|
-
!this._peerSessions.isReceiveAdmissionOpen(
|
|
19762
|
-
receiveFromHash,
|
|
19763
|
-
receiveSession,
|
|
19764
|
-
replicationLifecycleController,
|
|
19765
|
-
)
|
|
19766
|
-
) {
|
|
19767
|
-
return;
|
|
19768
|
-
}
|
|
19769
|
-
|
|
19770
|
-
let replicationSegments: ReplicationRangeIndexable<R>[];
|
|
19771
|
-
try {
|
|
19772
|
-
replicationSegments = await this.getMyReplicationSegments();
|
|
19773
|
-
} catch (error) {
|
|
19774
|
-
if (
|
|
19775
|
-
!this._peerSessions.isReceiveAdmissionOpen(
|
|
19776
|
-
receiveFromHash,
|
|
19777
|
-
receiveSession,
|
|
19778
|
-
replicationLifecycleController,
|
|
19779
|
-
) &&
|
|
19780
|
-
isNotStartedError(error as Error)
|
|
19781
|
-
) {
|
|
19782
|
-
return;
|
|
19783
|
-
}
|
|
19784
|
-
throw error;
|
|
19785
|
-
}
|
|
19786
|
-
if (
|
|
19787
|
-
!this._peerSessions.isReceiveAdmissionOpen(
|
|
19788
|
-
receiveFromHash,
|
|
19789
|
-
receiveSession,
|
|
19790
|
-
replicationLifecycleController,
|
|
19791
|
-
)
|
|
19792
|
-
) {
|
|
19793
|
-
return;
|
|
19794
|
-
}
|
|
19795
|
-
const segments = replicationSegments.map((x) => x.toReplicationRange());
|
|
19796
|
-
this.validatePersistedReplicationRangeSnapshot(segments);
|
|
19797
|
-
this._v2Send.enqueueSnapshotForPeer(receiveFromHash);
|
|
19798
|
-
|
|
19799
|
-
await this.rpc
|
|
19800
|
-
.send(new AllReplicatingSegmentsMessage({ segments }), {
|
|
19801
|
-
mode: new AcknowledgeDelivery({
|
|
19802
|
-
to: [context.from],
|
|
19803
|
-
redundancy: 1,
|
|
19804
|
-
}),
|
|
19805
|
-
signal: replicationLifecycleController.signal,
|
|
19806
|
-
})
|
|
19807
|
-
.catch((error) =>
|
|
19808
|
-
this.handleReplicationLifecycleSendError(
|
|
19809
|
-
error,
|
|
19810
|
-
replicationLifecycleController,
|
|
19811
|
-
),
|
|
19812
|
-
);
|
|
19813
|
-
if (
|
|
19814
|
-
!this._peerSessions.isReceiveAdmissionOpen(
|
|
19815
|
-
receiveFromHash,
|
|
19816
|
-
receiveSession,
|
|
19817
|
-
replicationLifecycleController,
|
|
19818
|
-
)
|
|
19819
|
-
) {
|
|
19820
|
-
return;
|
|
19821
|
-
}
|
|
19822
|
-
|
|
19823
|
-
// for backwards compatibility (v8) remove this when we are sure that all nodes are v9+
|
|
19824
|
-
if (this.v8Behaviour) {
|
|
19825
|
-
const role = this.getRoleFromReplicationSegments(replicationSegments);
|
|
19826
|
-
if (role instanceof Replicator) {
|
|
19827
|
-
const fixedSettings = !this._isAdaptiveReplicating;
|
|
19828
|
-
if (fixedSettings) {
|
|
19829
|
-
await this.rpc
|
|
19830
|
-
.send(
|
|
19831
|
-
new ResponseRoleMessage({
|
|
19832
|
-
role,
|
|
19833
|
-
}),
|
|
19834
|
-
{
|
|
19835
|
-
mode: new SilentDelivery({
|
|
19836
|
-
to: [context.from],
|
|
19837
|
-
redundancy: 1,
|
|
19838
|
-
}),
|
|
19839
|
-
signal: replicationLifecycleController.signal,
|
|
19840
|
-
},
|
|
19841
|
-
)
|
|
19842
|
-
.catch((error) =>
|
|
19843
|
-
this.handleReplicationLifecycleSendError(
|
|
19844
|
-
error,
|
|
19845
|
-
replicationLifecycleController,
|
|
19846
|
-
),
|
|
19847
|
-
);
|
|
19848
|
-
}
|
|
19849
|
-
}
|
|
19850
|
-
}
|
|
19851
|
-
}
|
|
19852
|
-
|
|
19853
19579
|
private async handleReplicationInfoV2Announcement(
|
|
19854
19580
|
msg: ReplicationInfoV2Message,
|
|
19855
19581
|
context: ReceiveRequestContext,
|
|
@@ -20029,251 +19755,12 @@ export class SharedLog<
|
|
|
20029
19755
|
// A committed V2 announcement is applied progress: the peer answers,
|
|
20030
19756
|
// so recovery re-solicitation may restart from the base interval.
|
|
20031
19757
|
this.resetReplicationInfoV2RecoveryEscalation(fromHash);
|
|
20032
|
-
if (
|
|
20033
|
-
msg instanceof FullReplicationInfoV2Message &&
|
|
20034
|
-
this.legacyReplicationInfoEnabled
|
|
20035
|
-
) {
|
|
20036
|
-
this.cancelReplicationInfoRequests(fromHash);
|
|
20037
|
-
}
|
|
20038
19758
|
});
|
|
20039
19759
|
} finally {
|
|
20040
19760
|
this._v2Receive.release(admission);
|
|
20041
19761
|
}
|
|
20042
19762
|
}
|
|
20043
19763
|
|
|
20044
|
-
private async handleReplicationInfoAnnouncement(
|
|
20045
|
-
msg: AllReplicatingSegmentsMessage | AddedReplicationSegmentMessage,
|
|
20046
|
-
context: ReceiveRequestContext,
|
|
20047
|
-
lane: ReceiveLaneContext,
|
|
20048
|
-
): Promise<void> {
|
|
20049
|
-
const receiveFromHash = lane.fromHash;
|
|
20050
|
-
const receiveSession = lane.session;
|
|
20051
|
-
const receiveReplicationLifecycleController = lane.lifecycleController;
|
|
20052
|
-
const receiveReplicationInfoReceiveEpoch = lane.receiveEpoch;
|
|
20053
|
-
const peerReceiveLease = lane.lease;
|
|
20054
|
-
if (context.from.equals(this.node.identity.publicKey)) {
|
|
20055
|
-
return;
|
|
20056
|
-
}
|
|
20057
|
-
|
|
20058
|
-
const replicationInfoMessage = msg as
|
|
20059
|
-
| AllReplicatingSegmentsMessage
|
|
20060
|
-
| AddedReplicationSegmentMessage;
|
|
20061
|
-
|
|
20062
|
-
// Process replication updates even if the sender isn't yet considered "ready" by
|
|
20063
|
-
// `Program.waitFor()`. Dropping these messages can lead to missing replicator info
|
|
20064
|
-
// (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
|
|
20065
|
-
const from = context.from!;
|
|
20066
|
-
const fromHash = from.hashcode();
|
|
20067
|
-
if (this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
20068
|
-
if (receiveSession) {
|
|
20069
|
-
this._v2Receive.noteLegacyAnnouncement({
|
|
20070
|
-
peerHash: fromHash,
|
|
20071
|
-
peerSession: receiveSession,
|
|
20072
|
-
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
20073
|
-
senderTransportSession: context.message.header.session,
|
|
20074
|
-
transportTimestamp: context.message.header.timestamp,
|
|
20075
|
-
message: msg,
|
|
20076
|
-
});
|
|
20077
|
-
}
|
|
20078
|
-
return;
|
|
20079
|
-
}
|
|
20080
|
-
// Pre-lane gate: lifecycle -> receive-epoch -> blocked, exactly the
|
|
20081
|
-
// legacy order. isMembershipActiveFor is the unit-pinned fold of
|
|
20082
|
-
// isReplicationLifecycleActive; isReceiveEpochCurrent is the
|
|
20083
|
-
// relocated `===`-with-`?? null`. The DELIBERATE absence of a
|
|
20084
|
-
// subscription-epoch term is preserved: the lease already validated
|
|
20085
|
-
// it, and the in-lane recheck owns post-await staleness.
|
|
20086
|
-
if (
|
|
20087
|
-
!this._instanceLifecycle!.isMembershipActiveFor(
|
|
20088
|
-
receiveReplicationLifecycleController,
|
|
20089
|
-
) ||
|
|
20090
|
-
!this._peerSessions.isReceiveEpochCurrent(
|
|
20091
|
-
receiveFromHash,
|
|
20092
|
-
receiveReplicationInfoReceiveEpoch,
|
|
20093
|
-
) ||
|
|
20094
|
-
this._peerSessions.isReplicationInfoBlocked(fromHash)
|
|
20095
|
-
) {
|
|
20096
|
-
return;
|
|
20097
|
-
}
|
|
20098
|
-
const messageTimestamp = context.message.header.timestamp;
|
|
20099
|
-
peerReceiveLease.release();
|
|
20100
|
-
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
20101
|
-
try {
|
|
20102
|
-
// The peer may have unsubscribed after this message was queued.
|
|
20103
|
-
// In-lane gate: lifecycle -> subscription-epoch -> receive-epoch
|
|
20104
|
-
// -> blocked, term for term as before the session migration.
|
|
20105
|
-
if (
|
|
20106
|
-
!this._instanceLifecycle!.isMembershipActiveFor(
|
|
20107
|
-
receiveReplicationLifecycleController,
|
|
20108
|
-
) ||
|
|
20109
|
-
!this._peerSessions.isCurrent(fromHash, receiveSession) ||
|
|
20110
|
-
!this._peerSessions.isReceiveEpochCurrent(
|
|
20111
|
-
fromHash,
|
|
20112
|
-
receiveReplicationInfoReceiveEpoch,
|
|
20113
|
-
) ||
|
|
20114
|
-
this._peerSessions.isReplicationInfoBlocked(fromHash)
|
|
20115
|
-
) {
|
|
20116
|
-
return;
|
|
20117
|
-
}
|
|
20118
|
-
if (receiveSession && this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
20119
|
-
this._v2Receive.noteLegacyAnnouncement({
|
|
20120
|
-
peerHash: fromHash,
|
|
20121
|
-
peerSession: receiveSession,
|
|
20122
|
-
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
20123
|
-
senderTransportSession: context.message.header.session,
|
|
20124
|
-
transportTimestamp: context.message.header.timestamp,
|
|
20125
|
-
message: msg,
|
|
20126
|
-
});
|
|
20127
|
-
return;
|
|
20128
|
-
}
|
|
20129
|
-
|
|
20130
|
-
// Process in-order to avoid races where repeated reset messages arrive
|
|
20131
|
-
// concurrently and trigger spurious "added" diffs / rebalancing.
|
|
20132
|
-
const prev = this.latestReplicationInfoMessage.get(fromHash);
|
|
20133
|
-
if (prev && prev > messageTimestamp) {
|
|
20134
|
-
return;
|
|
20135
|
-
}
|
|
20136
|
-
|
|
20137
|
-
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
20138
|
-
|
|
20139
|
-
if (this.closed) {
|
|
20140
|
-
return;
|
|
20141
|
-
}
|
|
20142
|
-
|
|
20143
|
-
const reset = msg instanceof AllReplicatingSegmentsMessage;
|
|
20144
|
-
const result = await this.addReplicationRange(
|
|
20145
|
-
replicationInfoMessage.segments.map((x) =>
|
|
20146
|
-
x.toReplicationRangeIndexable(from),
|
|
20147
|
-
),
|
|
20148
|
-
from,
|
|
20149
|
-
{
|
|
20150
|
-
reset,
|
|
20151
|
-
checkDuplicates: true,
|
|
20152
|
-
timestamp: Number(messageTimestamp),
|
|
20153
|
-
allowLegacyOrderedReplacementPairs:
|
|
20154
|
-
msg instanceof AddedReplicationSegmentMessage,
|
|
20155
|
-
},
|
|
20156
|
-
);
|
|
20157
|
-
if (result === undefined) {
|
|
20158
|
-
return;
|
|
20159
|
-
}
|
|
20160
|
-
this._liveness.markReplicatorActivity(fromHash);
|
|
20161
|
-
|
|
20162
|
-
// If the peer reports any replication segments, stop re-requesting.
|
|
20163
|
-
// (Empty reports can be transient during startup.)
|
|
20164
|
-
if (replicationInfoMessage.segments.length > 0) {
|
|
20165
|
-
this.cancelReplicationInfoRequests(fromHash);
|
|
20166
|
-
}
|
|
20167
|
-
} catch (e) {
|
|
20168
|
-
if (isNotStartedError(e as Error)) {
|
|
20169
|
-
return;
|
|
20170
|
-
}
|
|
20171
|
-
logger.error(
|
|
20172
|
-
`Failed to apply replication settings from '${fromHash}': ${
|
|
20173
|
-
(e as any)?.message ?? e
|
|
20174
|
-
}`,
|
|
20175
|
-
);
|
|
20176
|
-
}
|
|
20177
|
-
});
|
|
20178
|
-
}
|
|
20179
|
-
|
|
20180
|
-
private async handleStoppedReplicating(
|
|
20181
|
-
msg: StoppedReplicating,
|
|
20182
|
-
context: ReceiveRequestContext,
|
|
20183
|
-
lane: ReceiveLaneContext,
|
|
20184
|
-
): Promise<void> {
|
|
20185
|
-
const receiveFromHash = lane.fromHash;
|
|
20186
|
-
const receiveSession = lane.session;
|
|
20187
|
-
const receiveReplicationLifecycleController = lane.lifecycleController;
|
|
20188
|
-
const receiveReplicationInfoReceiveEpoch = lane.receiveEpoch;
|
|
20189
|
-
const peerReceiveLease = lane.lease;
|
|
20190
|
-
const from = context.from!;
|
|
20191
|
-
const segmentIds = msg.segmentIds;
|
|
20192
|
-
if (from.equals(this.node.identity.publicKey)) {
|
|
20193
|
-
return;
|
|
20194
|
-
}
|
|
20195
|
-
const fromHash = from.hashcode();
|
|
20196
|
-
if (this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
20197
|
-
if (receiveSession) {
|
|
20198
|
-
this._v2Receive.noteLegacyAnnouncement({
|
|
20199
|
-
peerHash: fromHash,
|
|
20200
|
-
peerSession: receiveSession,
|
|
20201
|
-
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
20202
|
-
senderTransportSession: context.message.header.session,
|
|
20203
|
-
transportTimestamp: context.message.header.timestamp,
|
|
20204
|
-
message: msg,
|
|
20205
|
-
});
|
|
20206
|
-
}
|
|
20207
|
-
return;
|
|
20208
|
-
}
|
|
20209
|
-
// Same pre-lane gate shape as Added/All above (and the same
|
|
20210
|
-
// intentional absence of a subscription-epoch term).
|
|
20211
|
-
if (
|
|
20212
|
-
!this._instanceLifecycle!.isMembershipActiveFor(
|
|
20213
|
-
receiveReplicationLifecycleController,
|
|
20214
|
-
) ||
|
|
20215
|
-
!this._peerSessions.isReceiveEpochCurrent(
|
|
20216
|
-
receiveFromHash,
|
|
20217
|
-
receiveReplicationInfoReceiveEpoch,
|
|
20218
|
-
) ||
|
|
20219
|
-
this._peerSessions.isReplicationInfoBlocked(fromHash)
|
|
20220
|
-
) {
|
|
20221
|
-
return;
|
|
20222
|
-
}
|
|
20223
|
-
const messageTimestamp = context.message.header.timestamp;
|
|
20224
|
-
peerReceiveLease.release();
|
|
20225
|
-
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
20226
|
-
if (
|
|
20227
|
-
!this._instanceLifecycle!.isMembershipActiveFor(
|
|
20228
|
-
receiveReplicationLifecycleController,
|
|
20229
|
-
) ||
|
|
20230
|
-
!this._peerSessions.isCurrent(fromHash, receiveSession) ||
|
|
20231
|
-
!this._peerSessions.isReceiveEpochCurrent(
|
|
20232
|
-
fromHash,
|
|
20233
|
-
receiveReplicationInfoReceiveEpoch,
|
|
20234
|
-
) ||
|
|
20235
|
-
this._peerSessions.isReplicationInfoBlocked(fromHash)
|
|
20236
|
-
) {
|
|
20237
|
-
return;
|
|
20238
|
-
}
|
|
20239
|
-
if (receiveSession && this._v2Receive.isLegacyCutover(receiveSession)) {
|
|
20240
|
-
this._v2Receive.noteLegacyAnnouncement({
|
|
20241
|
-
peerHash: fromHash,
|
|
20242
|
-
peerSession: receiveSession,
|
|
20243
|
-
receiveEpoch: receiveReplicationInfoReceiveEpoch,
|
|
20244
|
-
senderTransportSession: context.message.header.session,
|
|
20245
|
-
transportTimestamp: context.message.header.timestamp,
|
|
20246
|
-
message: msg,
|
|
20247
|
-
});
|
|
20248
|
-
return;
|
|
20249
|
-
}
|
|
20250
|
-
|
|
20251
|
-
const previousTimestamp = this.latestReplicationInfoMessage.get(fromHash);
|
|
20252
|
-
if (previousTimestamp && previousTimestamp > messageTimestamp) {
|
|
20253
|
-
return;
|
|
20254
|
-
}
|
|
20255
|
-
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
20256
|
-
if (this.closed) {
|
|
20257
|
-
return;
|
|
20258
|
-
}
|
|
20259
|
-
|
|
20260
|
-
const rangesToRemove = await this.resolveReplicationRangesFromIdsAndKey(
|
|
20261
|
-
segmentIds,
|
|
20262
|
-
from,
|
|
20263
|
-
);
|
|
20264
|
-
|
|
20265
|
-
await this.removeReplicationRanges(rangesToRemove, from);
|
|
20266
|
-
this._liveness.markReplicatorActivity(fromHash);
|
|
20267
|
-
const timestamp = BigInt(+new Date());
|
|
20268
|
-
for (const range of rangesToRemove) {
|
|
20269
|
-
this.replicationChangeDebounceFn.add({
|
|
20270
|
-
range,
|
|
20271
|
-
type: "removed",
|
|
20272
|
-
timestamp,
|
|
20273
|
-
});
|
|
20274
|
-
}
|
|
20275
|
-
});
|
|
20276
|
-
}
|
|
20277
19764
|
|
|
20278
19765
|
async calculateTotalParticipation(options?: { sum?: boolean }) {
|
|
20279
19766
|
if (options?.sum) {
|
|
@@ -20736,34 +20223,20 @@ export class SharedLog<
|
|
|
20736
20223
|
|
|
20737
20224
|
requestAttempts++;
|
|
20738
20225
|
|
|
20739
|
-
|
|
20740
|
-
|
|
20741
|
-
|
|
20742
|
-
|
|
20743
|
-
|
|
20744
|
-
|
|
20745
|
-
|
|
20746
|
-
|
|
20747
|
-
|
|
20748
|
-
|
|
20749
|
-
|
|
20750
|
-
|
|
20751
|
-
|
|
20752
|
-
|
|
20753
|
-
const peerSession = this._peerSessions.current(peerHash);
|
|
20754
|
-
if (peerSession?.phase === "open") {
|
|
20755
|
-
this._v2Receive.resumeParkedRequest({
|
|
20756
|
-
peerHash,
|
|
20757
|
-
peerSession,
|
|
20758
|
-
receiveEpoch: this._peerSessions.receiveEpoch(peerHash),
|
|
20759
|
-
});
|
|
20760
|
-
} else if (peerSession === null || peerSession.phase === "departing") {
|
|
20761
|
-
// A peer can be known to routing before its SharedLog topic
|
|
20762
|
-
// subscription has been observed. Legacy requests used to bootstrap
|
|
20763
|
-
// that case directly; V2 needs an authoritative Subscribe snapshot
|
|
20764
|
-
// before it can create a fenced PeerSession and request a Full.
|
|
20765
|
-
requestSubscriberSnapshot();
|
|
20766
|
-
}
|
|
20226
|
+
const peerHash = key.hashcode();
|
|
20227
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
20228
|
+
if (peerSession?.phase === "open") {
|
|
20229
|
+
this._v2Receive.resumeParkedRequest({
|
|
20230
|
+
peerHash,
|
|
20231
|
+
peerSession,
|
|
20232
|
+
receiveEpoch: this._peerSessions.receiveEpoch(peerHash),
|
|
20233
|
+
});
|
|
20234
|
+
} else if (peerSession === null || peerSession.phase === "departing") {
|
|
20235
|
+
// A peer can be known to routing before its SharedLog topic
|
|
20236
|
+
// subscription has been observed. Legacy requests used to bootstrap
|
|
20237
|
+
// that case directly; V2 needs an authoritative Subscribe snapshot
|
|
20238
|
+
// before it can create a fenced PeerSession and request a Full.
|
|
20239
|
+
requestSubscriberSnapshot();
|
|
20767
20240
|
}
|
|
20768
20241
|
|
|
20769
20242
|
if (requestAttempts < maxRequestAttempts) {
|
|
@@ -23507,7 +22980,7 @@ export class SharedLog<
|
|
|
23507
22980
|
*/
|
|
23508
22981
|
private resetReplicationInfoV2RecoveryEscalation(peerHash: string) {
|
|
23509
22982
|
const state = this._replicationInfoRequestByPeer.get(peerHash);
|
|
23510
|
-
if (!state
|
|
22983
|
+
if (!state) {
|
|
23511
22984
|
return;
|
|
23512
22985
|
}
|
|
23513
22986
|
state.attempts = 0;
|
|
@@ -23624,6 +23097,13 @@ export class SharedLog<
|
|
|
23624
23097
|
tick();
|
|
23625
23098
|
}
|
|
23626
23099
|
|
|
23100
|
+
/**
|
|
23101
|
+
* Collapsed B12 shell: the legacy request-polling body (bounded
|
|
23102
|
+
* RequestReplicationInfoMessage ticks) is deleted; V2 recovery is the
|
|
23103
|
+
* only scheduler. Retained as a named seam rather than inlined at the
|
|
23104
|
+
* callers because the liveness monitor wiring and several suites
|
|
23105
|
+
* stub/spy it by name.
|
|
23106
|
+
*/
|
|
23627
23107
|
private scheduleReplicationInfoRequests(
|
|
23628
23108
|
peer: PublicSignKey,
|
|
23629
23109
|
replicationLifecycleController = this._instanceLifecycle
|
|
@@ -23635,75 +23115,10 @@ export class SharedLog<
|
|
|
23635
23115
|
) {
|
|
23636
23116
|
return;
|
|
23637
23117
|
}
|
|
23638
|
-
|
|
23639
|
-
|
|
23640
|
-
|
|
23641
|
-
|
|
23642
|
-
);
|
|
23643
|
-
return;
|
|
23644
|
-
}
|
|
23645
|
-
const peerHash = peer.hashcode();
|
|
23646
|
-
const requestStates = this._replicationInfoRequestByPeer;
|
|
23647
|
-
if (requestStates.has(peerHash)) {
|
|
23648
|
-
return;
|
|
23649
|
-
}
|
|
23650
|
-
|
|
23651
|
-
const state: { attempts: number; timer?: ReturnType<typeof setTimeout> } = {
|
|
23652
|
-
attempts: 0,
|
|
23653
|
-
};
|
|
23654
|
-
requestStates.set(peerHash, state);
|
|
23655
|
-
const cancel = () => {
|
|
23656
|
-
if (requestStates.get(peerHash) !== state) {
|
|
23657
|
-
return;
|
|
23658
|
-
}
|
|
23659
|
-
if (state.timer) {
|
|
23660
|
-
clearTimeout(state.timer);
|
|
23661
|
-
}
|
|
23662
|
-
requestStates.delete(peerHash);
|
|
23663
|
-
};
|
|
23664
|
-
|
|
23665
|
-
const intervalMs = Math.max(50, this.waitForReplicatorRequestIntervalMs);
|
|
23666
|
-
const maxAttempts =
|
|
23667
|
-
this.waitForReplicatorRequestMaxAttempts ??
|
|
23668
|
-
Math.max(
|
|
23669
|
-
WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS,
|
|
23670
|
-
Math.ceil(this.waitForReplicatorTimeout / intervalMs),
|
|
23671
|
-
);
|
|
23672
|
-
|
|
23673
|
-
const tick = () => {
|
|
23674
|
-
if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
23675
|
-
cancel();
|
|
23676
|
-
return;
|
|
23677
|
-
}
|
|
23678
|
-
state.attempts++;
|
|
23679
|
-
|
|
23680
|
-
this.rpc
|
|
23681
|
-
.send(new RequestReplicationInfoMessage(), {
|
|
23682
|
-
mode: new AcknowledgeDelivery({ redundancy: 1, to: [peer] }),
|
|
23683
|
-
signal: replicationLifecycleController.signal,
|
|
23684
|
-
})
|
|
23685
|
-
.catch((e) => {
|
|
23686
|
-
// Best-effort: missing peers / unopened RPC should not fail join flows.
|
|
23687
|
-
if (
|
|
23688
|
-
isNotStartedError(e as Error) ||
|
|
23689
|
-
(replicationLifecycleController.signal.aborted &&
|
|
23690
|
-
e instanceof AbortError)
|
|
23691
|
-
) {
|
|
23692
|
-
return;
|
|
23693
|
-
}
|
|
23694
|
-
logger.error(e?.toString?.() ?? String(e));
|
|
23695
|
-
});
|
|
23696
|
-
|
|
23697
|
-
if (state.attempts >= maxAttempts) {
|
|
23698
|
-
cancel();
|
|
23699
|
-
return;
|
|
23700
|
-
}
|
|
23701
|
-
|
|
23702
|
-
state.timer = setTimeout(tick, intervalMs);
|
|
23703
|
-
state.timer.unref?.();
|
|
23704
|
-
};
|
|
23705
|
-
|
|
23706
|
-
tick();
|
|
23118
|
+
this.scheduleReplicationInfoV2Recovery(
|
|
23119
|
+
peer,
|
|
23120
|
+
replicationLifecycleController,
|
|
23121
|
+
);
|
|
23707
23122
|
}
|
|
23708
23123
|
|
|
23709
23124
|
async handleSubscriptionChange(
|
|
@@ -23786,10 +23201,6 @@ export class SharedLog<
|
|
|
23786
23201
|
) {
|
|
23787
23202
|
return;
|
|
23788
23203
|
}
|
|
23789
|
-
// The timestamp watermark belongs to the previous subscription epoch.
|
|
23790
|
-
// Sender clocks are not synchronized, so carrying a local unsubscribe
|
|
23791
|
-
// timestamp forward could reject every valid announcement after reconnect.
|
|
23792
|
-
this.latestReplicationInfoMessage.delete(peerHash);
|
|
23793
23204
|
this._pendingReplicatorLeaveByPeer.delete(peerHash);
|
|
23794
23205
|
const openingCapabilities =
|
|
23795
23206
|
this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
@@ -23843,12 +23254,6 @@ export class SharedLog<
|
|
|
23843
23254
|
this.joinWarmup._warmupSessionsByTarget.get(peerHash) ?? null;
|
|
23844
23255
|
this.joinWarmup.cancelJoinWarmupTarget(peerHash);
|
|
23845
23256
|
|
|
23846
|
-
const now = BigInt(+new Date());
|
|
23847
|
-
const previous = this.latestReplicationInfoMessage.get(peerHash);
|
|
23848
|
-
if (!previous || previous < now) {
|
|
23849
|
-
this.latestReplicationInfoMessage.set(peerHash, now);
|
|
23850
|
-
}
|
|
23851
|
-
|
|
23852
23257
|
let removed = false;
|
|
23853
23258
|
try {
|
|
23854
23259
|
// Unsubscribe can race with the peer's final replication reset message.
|
|
@@ -23899,116 +23304,23 @@ export class SharedLog<
|
|
|
23899
23304
|
|
|
23900
23305
|
// Decode, sender and authenticated apply readiness are separate bits. An
|
|
23901
23306
|
// ACKed capability advert is the local half of receiver-led negotiation;
|
|
23902
|
-
// readiness is promoted
|
|
23903
|
-
// This
|
|
23904
|
-
//
|
|
23307
|
+
// readiness is promoted through the coordinator once the ACK arrives.
|
|
23308
|
+
// This keeps capability ACK latency off the subscription callback's
|
|
23309
|
+
// critical path.
|
|
23905
23310
|
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
23906
|
-
|
|
23907
|
-
|
|
23908
|
-
|
|
23909
|
-
|
|
23910
|
-
|
|
23911
|
-
|
|
23912
|
-
|
|
23913
|
-
|
|
23914
|
-
|
|
23915
|
-
|
|
23916
|
-
|
|
23917
|
-
|
|
23918
|
-
|
|
23919
|
-
publicKey,
|
|
23920
|
-
replicationLifecycleController,
|
|
23921
|
-
);
|
|
23922
|
-
return;
|
|
23923
|
-
}
|
|
23924
|
-
|
|
23925
|
-
try {
|
|
23926
|
-
let replicationSegments: ReplicationRangeIndexable<R>[];
|
|
23927
|
-
try {
|
|
23928
|
-
replicationSegments = await this.getMyReplicationSegments();
|
|
23929
|
-
} catch (error) {
|
|
23930
|
-
if (
|
|
23931
|
-
!this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
23932
|
-
isNotStartedError(error as Error)
|
|
23933
|
-
) {
|
|
23934
|
-
return;
|
|
23935
|
-
}
|
|
23936
|
-
throw error;
|
|
23937
|
-
}
|
|
23938
|
-
if (
|
|
23939
|
-
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
23940
|
-
!ownsSubscriptionEpoch()
|
|
23941
|
-
) {
|
|
23942
|
-
return;
|
|
23943
|
-
}
|
|
23944
|
-
if (replicationSegments.length > 0) {
|
|
23945
|
-
const segments = replicationSegments.map((x) => x.toReplicationRange());
|
|
23946
|
-
this.validatePersistedReplicationRangeSnapshot(segments);
|
|
23947
|
-
await this.rpc
|
|
23948
|
-
.send(
|
|
23949
|
-
new AllReplicatingSegmentsMessage({
|
|
23950
|
-
segments,
|
|
23951
|
-
}),
|
|
23952
|
-
{
|
|
23953
|
-
mode: new AcknowledgeDelivery({
|
|
23954
|
-
redundancy: 1,
|
|
23955
|
-
to: [publicKey],
|
|
23956
|
-
}),
|
|
23957
|
-
signal: replicationLifecycleController.signal,
|
|
23958
|
-
},
|
|
23959
|
-
)
|
|
23960
|
-
.catch((error) =>
|
|
23961
|
-
this.handleReplicationLifecycleSendError(
|
|
23962
|
-
error,
|
|
23963
|
-
replicationLifecycleController,
|
|
23964
|
-
),
|
|
23965
|
-
);
|
|
23966
|
-
if (
|
|
23967
|
-
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
23968
|
-
!ownsSubscriptionEpoch()
|
|
23969
|
-
) {
|
|
23970
|
-
return;
|
|
23971
|
-
}
|
|
23972
|
-
|
|
23973
|
-
if (this.v8Behaviour) {
|
|
23974
|
-
// for backwards compatibility
|
|
23975
|
-
await this.rpc
|
|
23976
|
-
.send(
|
|
23977
|
-
new ResponseRoleMessage({
|
|
23978
|
-
role: this.getRoleFromReplicationSegments(replicationSegments),
|
|
23979
|
-
}),
|
|
23980
|
-
{
|
|
23981
|
-
mode: new AcknowledgeDelivery({
|
|
23982
|
-
redundancy: 1,
|
|
23983
|
-
to: [publicKey],
|
|
23984
|
-
}),
|
|
23985
|
-
signal: replicationLifecycleController.signal,
|
|
23986
|
-
},
|
|
23987
|
-
)
|
|
23988
|
-
.catch((error) =>
|
|
23989
|
-
this.handleReplicationLifecycleSendError(
|
|
23990
|
-
error,
|
|
23991
|
-
replicationLifecycleController,
|
|
23992
|
-
),
|
|
23993
|
-
);
|
|
23994
|
-
}
|
|
23995
|
-
}
|
|
23996
|
-
|
|
23997
|
-
// Keep legacy request-based discovery independent of the capability ACK.
|
|
23998
|
-
// This makes mixed-version joins resilient to timing-sensitive delivery/order
|
|
23999
|
-
// issues where we may miss the remote peer's initial announcement.
|
|
24000
|
-
if (
|
|
24001
|
-
this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
24002
|
-
ownsSubscriptionEpoch()
|
|
24003
|
-
) {
|
|
24004
|
-
this.scheduleReplicationInfoRequests(
|
|
24005
|
-
publicKey,
|
|
24006
|
-
replicationLifecycleController,
|
|
24007
|
-
);
|
|
24008
|
-
}
|
|
24009
|
-
} finally {
|
|
24010
|
-
localCapabilityAdvertisement.releaseLegacyBarrier();
|
|
24011
|
-
}
|
|
23311
|
+
// B12: there is no legacy startup work to order ahead of RequestV2, so
|
|
23312
|
+
// the two-phase legacy barrier is gone — an ACKed advert promotes
|
|
23313
|
+
// readiness through the coordinator as soon as it lands.
|
|
23314
|
+
this._v2Receive.advertiseLocalCapability({
|
|
23315
|
+
target: publicKey,
|
|
23316
|
+
peerSession: expectedSubscriptionEpoch,
|
|
23317
|
+
receiveEpoch,
|
|
23318
|
+
signal: replicationLifecycleController.signal,
|
|
23319
|
+
});
|
|
23320
|
+
this.scheduleReplicationInfoV2Recovery(
|
|
23321
|
+
publicKey,
|
|
23322
|
+
replicationLifecycleController,
|
|
23323
|
+
);
|
|
24012
23324
|
}
|
|
24013
23325
|
|
|
24014
23326
|
private getClampedReplicas(customValue?: MinReplicas) {
|
|
@@ -25409,15 +24721,6 @@ export class SharedLog<
|
|
|
25409
24721
|
const subscriptionEpoch = this._peerSessions.rotate(fromHash, "departing");
|
|
25410
24722
|
this._peerSessions.blockReplicationInfo(fromHash);
|
|
25411
24723
|
this._recentRepairDispatch.delete(fromHash);
|
|
25412
|
-
|
|
25413
|
-
// Keep a per-peer timestamp watermark when we observe an unsubscribe. This
|
|
25414
|
-
// prevents late/out-of-order replication-info messages from re-introducing
|
|
25415
|
-
// stale segments for a peer that has already left the topic.
|
|
25416
|
-
const now = BigInt(+new Date());
|
|
25417
|
-
const prev = this.latestReplicationInfoMessage.get(fromHash);
|
|
25418
|
-
if (!prev || prev < now) {
|
|
25419
|
-
this.latestReplicationInfoMessage.set(fromHash, now);
|
|
25420
|
-
}
|
|
25421
24724
|
this.invalidateSharedLogTopicSubscribersCache();
|
|
25422
24725
|
|
|
25423
24726
|
return this.handleSubscriptionChange(
|
|
@@ -25581,26 +24884,16 @@ export class SharedLog<
|
|
|
25581
24884
|
return false;
|
|
25582
24885
|
}
|
|
25583
24886
|
|
|
25584
|
-
|
|
25585
|
-
|
|
25586
|
-
|
|
25587
|
-
|
|
25588
|
-
|
|
25589
|
-
|
|
25590
|
-
|
|
25591
|
-
|
|
25592
|
-
|
|
25593
|
-
|
|
25594
|
-
if (!isCurrent()) return false;
|
|
25595
|
-
} catch (error) {
|
|
25596
|
-
if (
|
|
25597
|
-
isTransientReplicationAnnouncementError(error) &&
|
|
25598
|
-
this._announcements._replicationAnnouncementRetryPending
|
|
25599
|
-
) {
|
|
25600
|
-
return false;
|
|
25601
|
-
}
|
|
25602
|
-
throw error;
|
|
25603
|
-
}
|
|
24887
|
+
await this.startAnnounceReplicating(
|
|
24888
|
+
[dynamicRange],
|
|
24889
|
+
{
|
|
24890
|
+
checkDuplicates: false,
|
|
24891
|
+
reset: false,
|
|
24892
|
+
shouldApply: isCurrent,
|
|
24893
|
+
},
|
|
24894
|
+
ownershipLifecycleController,
|
|
24895
|
+
);
|
|
24896
|
+
if (!isCurrent()) return false;
|
|
25604
24897
|
|
|
25605
24898
|
/* await this._updateRole(newRole, onRoleChange); */
|
|
25606
24899
|
if (isCurrent()) {
|