@peerbit/shared-log 13.2.34 → 13.2.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +26 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +307 -38
- package/dist/src/index.js.map +1 -1
- package/dist/src/peer-session.d.ts +2 -1
- package/dist/src/peer-session.d.ts.map +1 -1
- package/dist/src/peer-session.js +8 -2
- package/dist/src/peer-session.js.map +1 -1
- package/dist/src/replication-announcement.d.ts +1 -0
- package/dist/src/replication-announcement.d.ts.map +1 -1
- package/dist/src/replication-announcement.js +16 -2
- package/dist/src/replication-announcement.js.map +1 -1
- package/dist/src/replication-info-v2-receive.d.ts +21 -0
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -1
- package/dist/src/replication-info-v2-receive.js +106 -5
- package/dist/src/replication-info-v2-receive.js.map +1 -1
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +22 -1
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/dist/src/sync/factory.js +1 -1
- package/dist/src/sync/factory.js.map +1 -1
- package/package.json +13 -13
- package/src/index.ts +371 -48
- package/src/peer-session.ts +8 -0
- package/src/replication-announcement.ts +17 -2
- package/src/replication-info-v2-receive.ts +135 -5
- package/src/replication-info-v2-send.ts +21 -1
- package/src/sync/factory.ts +1 -1
package/src/index.ts
CHANGED
|
@@ -1346,6 +1346,13 @@ export const WAIT_FOR_REPLICATOR_TIMEOUT = 20000;
|
|
|
1346
1346
|
export const WAIT_FOR_ROLE_MATURITY = 5000;
|
|
1347
1347
|
export const WAIT_FOR_REPLICATOR_REQUEST_INTERVAL = 1000;
|
|
1348
1348
|
export const WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS = 3;
|
|
1349
|
+
// The V2 recovery scheduler is deliberately persistent (a subscribed peer is
|
|
1350
|
+
// re-solicited for as long as its topic session stays open), but consecutive
|
|
1351
|
+
// fruitless park/unpark cycles double the wait before the next unpark so a
|
|
1352
|
+
// silent-but-subscribed peer converges to one bounded request cycle per cap
|
|
1353
|
+
// window instead of one per base interval. Any applied V2 progress resets it.
|
|
1354
|
+
export const REPLICATION_INFO_V2_RECOVERY_MAX_UNPARK_DELAY = 300_000;
|
|
1355
|
+
const REPLICATION_INFO_V2_RECOVERY_MAX_UNPARK_EXPONENT = 20;
|
|
1349
1356
|
// TODO(prune): Investigate if/when a non-zero prune delay is required for correctness
|
|
1350
1357
|
// (e.g. responsibility/replication-info message reordering in multi-peer scenarios).
|
|
1351
1358
|
// Prefer making pruning robust without timing-based heuristics.
|
|
@@ -2086,9 +2093,23 @@ export class SharedLog<
|
|
|
2086
2093
|
// reconnect barrier commits. See PeerSessionRegistry._replicationInfoBlockedPeers.
|
|
2087
2094
|
private _replicationInfoRequestByPeer!: Map<
|
|
2088
2095
|
string,
|
|
2089
|
-
{
|
|
2096
|
+
{
|
|
2097
|
+
// Legacy scheduler: sends issued (bounded by maxAttempts). V2 recovery
|
|
2098
|
+
// scheduler: consecutive fruitless unparks — the escalation exponent
|
|
2099
|
+
// for the next unpark delay, reset on any applied V2 progress.
|
|
2100
|
+
attempts: number;
|
|
2101
|
+
timer?: ReturnType<typeof setTimeout>;
|
|
2102
|
+
peerSession?: PeerSession;
|
|
2103
|
+
// V2 recovery scheduler only: when the current park was first observed.
|
|
2104
|
+
parkedSinceMs?: number;
|
|
2105
|
+
}
|
|
2090
2106
|
>;
|
|
2091
2107
|
private _replicationInfoApplyQueueByPeer!: Map<string, Promise<void>>;
|
|
2108
|
+
// One in-flight targeted subscriber-snapshot request per session-less peer.
|
|
2109
|
+
// A capability burst from a peer whose Subscribe has not been observed must
|
|
2110
|
+
// coalesce into a single pubsub.requestSubscribers call (mirrors the
|
|
2111
|
+
// waitForReplicator in-flight coalescing); a later burst may request again.
|
|
2112
|
+
private _subscriberSnapshotRequestsByPeer!: Map<string, Promise<void>>;
|
|
2092
2113
|
// Range ids are global primary keys while receive lanes are per peer. Keep
|
|
2093
2114
|
// reads and writes that decide one mutation in a single global lane.
|
|
2094
2115
|
private _replicationRangeMutationTail: Promise<void> = Promise.resolve();
|
|
@@ -3382,6 +3403,7 @@ export class SharedLog<
|
|
|
3382
3403
|
error,
|
|
3383
3404
|
),
|
|
3384
3405
|
enqueueReplicationInfoV2: (message) => this._v2Send.enqueue(message),
|
|
3406
|
+
isLegacyReplicationInfoEnabled: () => this.legacyReplicationInfoEnabled,
|
|
3385
3407
|
isClosed: () => this.closed,
|
|
3386
3408
|
getCloseSignal: () => this._closeController.signal,
|
|
3387
3409
|
getMyReplicationSegments: () => this.getMyReplicationSegments(),
|
|
@@ -3628,6 +3650,7 @@ export class SharedLog<
|
|
|
3628
3650
|
this._pendingIHaveCallbacks = new Set();
|
|
3629
3651
|
this.latestReplicationInfoMessage = new Map();
|
|
3630
3652
|
this._replicationInfoRequestByPeer = new Map();
|
|
3653
|
+
this._subscriberSnapshotRequestsByPeer = new Map();
|
|
3631
3654
|
this._replicationInfoApplyQueueByPeer = new Map();
|
|
3632
3655
|
// The registry constructor runs resetForOpen(), which creates the
|
|
3633
3656
|
// replication-info blocked set (fence B5) alongside the session maps —
|
|
@@ -3740,6 +3763,14 @@ export class SharedLog<
|
|
|
3740
3763
|
return this._logProperties?.compatibility;
|
|
3741
3764
|
}
|
|
3742
3765
|
|
|
3766
|
+
/**
|
|
3767
|
+
* Legacy replication-info is an explicit compatibility fallback. Current
|
|
3768
|
+
* logs never infer or re-enable it from a remote peer's capabilities.
|
|
3769
|
+
*/
|
|
3770
|
+
private get legacyReplicationInfoEnabled(): boolean {
|
|
3771
|
+
return this.compatibility !== undefined && this.compatibility < 10;
|
|
3772
|
+
}
|
|
3773
|
+
|
|
3743
3774
|
get isAdaptiveReplicating() {
|
|
3744
3775
|
return this._isAdaptiveReplicating;
|
|
3745
3776
|
}
|
|
@@ -4269,6 +4300,9 @@ export class SharedLog<
|
|
|
4269
4300
|
);
|
|
4270
4301
|
if (generationAdvanced) {
|
|
4271
4302
|
this._v2Send.advancePeerCapability(peerHash);
|
|
4303
|
+
// A fresh signed capability generation is V2 progress from the peer:
|
|
4304
|
+
// recovery re-solicitation may restart from the base interval.
|
|
4305
|
+
this.resetReplicationInfoV2RecoveryEscalation(peerHash);
|
|
4272
4306
|
}
|
|
4273
4307
|
return true;
|
|
4274
4308
|
}
|
|
@@ -4299,6 +4333,36 @@ export class SharedLog<
|
|
|
4299
4333
|
});
|
|
4300
4334
|
}
|
|
4301
4335
|
|
|
4336
|
+
/**
|
|
4337
|
+
* Coalesced targeted subscriber-snapshot request for the
|
|
4338
|
+
* capability-before-Subscribe recovery path. The observed-capability gate
|
|
4339
|
+
* is sender-paced (any advancing timestamp passes), so a burst of frames
|
|
4340
|
+
* from one session-less peer must not fan out into one GetSubscribers
|
|
4341
|
+
* unicast per frame. One request per peer is in flight at a time; once it
|
|
4342
|
+
* settles, a genuinely new session-less capability may request again.
|
|
4343
|
+
*/
|
|
4344
|
+
private requestSubscriberSnapshotForCapability(target: PublicSignKey): void {
|
|
4345
|
+
const peerHash = target.hashcode();
|
|
4346
|
+
if (this._subscriberSnapshotRequestsByPeer.has(peerHash)) {
|
|
4347
|
+
return;
|
|
4348
|
+
}
|
|
4349
|
+
const request = Promise.resolve()
|
|
4350
|
+
.then(() =>
|
|
4351
|
+
this.node.services.pubsub.requestSubscribers(this.topic, target),
|
|
4352
|
+
)
|
|
4353
|
+
.catch((error) => {
|
|
4354
|
+
if (!isNotStartedError(error as Error)) {
|
|
4355
|
+
logger.error(error?.toString?.() ?? String(error));
|
|
4356
|
+
}
|
|
4357
|
+
})
|
|
4358
|
+
.finally(() => {
|
|
4359
|
+
if (this._subscriberSnapshotRequestsByPeer.get(peerHash) === request) {
|
|
4360
|
+
this._subscriberSnapshotRequestsByPeer.delete(peerHash);
|
|
4361
|
+
}
|
|
4362
|
+
});
|
|
4363
|
+
this._subscriberSnapshotRequestsByPeer.set(peerHash, request);
|
|
4364
|
+
}
|
|
4365
|
+
|
|
4302
4366
|
/**
|
|
4303
4367
|
* Live append gossip may use the raw exchange-heads path only when we
|
|
4304
4368
|
* opted into raw sync and every remote recipient advertised raw capability
|
|
@@ -14163,7 +14227,9 @@ export class SharedLog<
|
|
|
14163
14227
|
this.domain = options?.domain
|
|
14164
14228
|
? (options.domain(this) as unknown as D)
|
|
14165
14229
|
: (createReplicationDomainHash(
|
|
14166
|
-
options?.compatibility && options
|
|
14230
|
+
options?.compatibility !== undefined && options.compatibility < 10
|
|
14231
|
+
? "u32"
|
|
14232
|
+
: "u64",
|
|
14167
14233
|
)(this) as unknown as D);
|
|
14168
14234
|
this.indexableDomain = createIndexableDomainFromResolution(
|
|
14169
14235
|
this.domain.resolution,
|
|
@@ -14176,6 +14242,7 @@ export class SharedLog<
|
|
|
14176
14242
|
this._pendingIHaveCallbacks = new Set();
|
|
14177
14243
|
this.latestReplicationInfoMessage = new Map();
|
|
14178
14244
|
this._replicationInfoRequestByPeer = new Map();
|
|
14245
|
+
this._subscriberSnapshotRequestsByPeer = new Map();
|
|
14179
14246
|
// Terminal close/drop drains the previous lifecycle before another open can
|
|
14180
14247
|
// install fresh lanes and opaque per-subscription ownership tokens.
|
|
14181
14248
|
this._replicationInfoApplyQueueByPeer = new Map();
|
|
@@ -14309,8 +14376,12 @@ export class SharedLog<
|
|
|
14309
14376
|
}
|
|
14310
14377
|
|
|
14311
14378
|
this._closeController = new AbortController();
|
|
14312
|
-
this.
|
|
14313
|
-
|
|
14379
|
+
if (this.legacyReplicationInfoEnabled) {
|
|
14380
|
+
this._announcements.setupReplicationAnnouncementRetryFunction();
|
|
14381
|
+
this._announcements.setupReplicationAnnouncementRepairFunction();
|
|
14382
|
+
} else {
|
|
14383
|
+
this._announcements.cancelCurrentReplicationStateAnnouncementRetry();
|
|
14384
|
+
}
|
|
14314
14385
|
this._closeController.signal.addEventListener("abort", () => {
|
|
14315
14386
|
for (const [_peer, state] of this._replicationInfoRequestByPeer) {
|
|
14316
14387
|
if (state.timer) clearTimeout(state.timer);
|
|
@@ -15512,7 +15583,14 @@ export class SharedLog<
|
|
|
15512
15583
|
|
|
15513
15584
|
async afterOpen(): Promise<void> {
|
|
15514
15585
|
await super.afterOpen();
|
|
15515
|
-
|
|
15586
|
+
// Start the broader discovery eagerly, in parallel with rebalance, for its
|
|
15587
|
+
// routing/cache side effects. It also contains connected/provider/fanout
|
|
15588
|
+
// candidates that have not subscribed to this log, so only the authoritative
|
|
15589
|
+
// pubsub snapshot below may create subscription fallback sessions.
|
|
15590
|
+
const subscriberDiscoveryPromise = this._getTopicSubscribers(this.topic);
|
|
15591
|
+
const existingSubscribersPromise = this.node.services.pubsub.getSubscribers(
|
|
15592
|
+
this.topic,
|
|
15593
|
+
);
|
|
15516
15594
|
const replicationLifecycleController =
|
|
15517
15595
|
this._instanceLifecycle?.membershipLifecycleController;
|
|
15518
15596
|
|
|
@@ -15533,6 +15611,7 @@ export class SharedLog<
|
|
|
15533
15611
|
this._liveness.startReplicatorLivenessSweep();
|
|
15534
15612
|
|
|
15535
15613
|
await this.rebalanceParticipation();
|
|
15614
|
+
await subscriberDiscoveryPromise;
|
|
15536
15615
|
|
|
15537
15616
|
// Take into account existing subscription
|
|
15538
15617
|
(await existingSubscribersPromise)?.forEach((v) => {
|
|
@@ -15542,6 +15621,14 @@ export class SharedLog<
|
|
|
15542
15621
|
if (this.closed) {
|
|
15543
15622
|
return;
|
|
15544
15623
|
}
|
|
15624
|
+
// The live subscribe event and this after-open snapshot can report the
|
|
15625
|
+
// same initial transport generation. The live callback rotates its
|
|
15626
|
+
// PeerSession synchronously, so any current session here proves the
|
|
15627
|
+
// fallback is stale/duplicate. Rotating again would erase the signed
|
|
15628
|
+
// capability binding that the first callback just established.
|
|
15629
|
+
if (this._peerSessions.current(v.hashcode()) !== null) {
|
|
15630
|
+
return;
|
|
15631
|
+
}
|
|
15545
15632
|
void this.runSubscriptionChangeCallback(() =>
|
|
15546
15633
|
this.handleSubscriptionChange(v, [this.topic], true),
|
|
15547
15634
|
);
|
|
@@ -15754,7 +15841,9 @@ export class SharedLog<
|
|
|
15754
15841
|
const peerSession = this._peerSessions.current(peerHash);
|
|
15755
15842
|
const preserveV2Session =
|
|
15756
15843
|
peerSession?.phase === "open" && peerSession.isActive();
|
|
15757
|
-
this.
|
|
15844
|
+
if (this.legacyReplicationInfoEnabled || !preserveV2Session) {
|
|
15845
|
+
this.cancelReplicationInfoRequests(peerHash);
|
|
15846
|
+
}
|
|
15758
15847
|
this._liveness._replicatorLivenessFailures.delete(peerHash);
|
|
15759
15848
|
this._liveness._replicatorLastActivityAt.delete(peerHash);
|
|
15760
15849
|
if (!preserveV2Session) {
|
|
@@ -16925,15 +17014,18 @@ export class SharedLog<
|
|
|
16925
17014
|
}, 2_000);
|
|
16926
17015
|
try {
|
|
16927
17016
|
const reset = new AllReplicatingSegmentsMessage({ segments: [] });
|
|
16928
|
-
|
|
16929
|
-
|
|
16930
|
-
|
|
16931
|
-
|
|
16932
|
-
|
|
16933
|
-
|
|
16934
|
-
|
|
16935
|
-
|
|
16936
|
-
|
|
17017
|
+
const resets = [this._v2Send.sendTerminalReset(abort.signal)];
|
|
17018
|
+
if (this.legacyReplicationInfoEnabled) {
|
|
17019
|
+
resets.push(
|
|
17020
|
+
this.rpc
|
|
17021
|
+
.send(reset, {
|
|
17022
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
17023
|
+
signal: abort.signal,
|
|
17024
|
+
})
|
|
17025
|
+
.catch(() => {}),
|
|
17026
|
+
);
|
|
17027
|
+
}
|
|
17028
|
+
await Promise.all(resets);
|
|
16937
17029
|
} finally {
|
|
16938
17030
|
clearTimeout(abortTimer);
|
|
16939
17031
|
}
|
|
@@ -17055,15 +17147,18 @@ export class SharedLog<
|
|
|
17055
17147
|
}, 2_000);
|
|
17056
17148
|
try {
|
|
17057
17149
|
const reset = new AllReplicatingSegmentsMessage({ segments: [] });
|
|
17058
|
-
|
|
17059
|
-
|
|
17060
|
-
|
|
17061
|
-
|
|
17062
|
-
|
|
17063
|
-
|
|
17064
|
-
|
|
17065
|
-
|
|
17066
|
-
|
|
17150
|
+
const resets = [this._v2Send.sendTerminalReset(abort.signal)];
|
|
17151
|
+
if (this.legacyReplicationInfoEnabled) {
|
|
17152
|
+
resets.push(
|
|
17153
|
+
this.rpc
|
|
17154
|
+
.send(reset, {
|
|
17155
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
17156
|
+
signal: abort.signal,
|
|
17157
|
+
})
|
|
17158
|
+
.catch(() => {}),
|
|
17159
|
+
);
|
|
17160
|
+
}
|
|
17161
|
+
await Promise.all(resets);
|
|
17067
17162
|
} finally {
|
|
17068
17163
|
clearTimeout(abortTimer);
|
|
17069
17164
|
}
|
|
@@ -17589,6 +17684,19 @@ export class SharedLog<
|
|
|
17589
17684
|
if (!context.from) {
|
|
17590
17685
|
throw new Error("Missing from in update role message");
|
|
17591
17686
|
}
|
|
17687
|
+
if (
|
|
17688
|
+
!this.legacyReplicationInfoEnabled &&
|
|
17689
|
+
(msg instanceof RequestReplicationInfoMessage ||
|
|
17690
|
+
msg instanceof ResponseRoleMessage ||
|
|
17691
|
+
msg instanceof AllReplicatingSegmentsMessage ||
|
|
17692
|
+
msg instanceof AddedReplicationSegmentMessage ||
|
|
17693
|
+
msg instanceof StoppedReplicating)
|
|
17694
|
+
) {
|
|
17695
|
+
// These variants remain registered decode tombstones, but current logs
|
|
17696
|
+
// fail closed before leases, synchronizer work, liveness, watermarks or
|
|
17697
|
+
// mutations. Only an explicit pre-v10 compatibility open admits them.
|
|
17698
|
+
return;
|
|
17699
|
+
}
|
|
17592
17700
|
// Snapshot receive ownership before any async handler gets a chance to
|
|
17593
17701
|
// yield. Replication-info messages reach their branch only after the
|
|
17594
17702
|
// synchronizer declines them, and a U/S transition can happen meanwhile.
|
|
@@ -19310,19 +19418,26 @@ export class SharedLog<
|
|
|
19310
19418
|
timestamp: capabilityTimestamp,
|
|
19311
19419
|
openingSession: receiveSession!,
|
|
19312
19420
|
});
|
|
19313
|
-
} else
|
|
19314
|
-
this.observePeerSyncCapabilities({
|
|
19421
|
+
} else {
|
|
19422
|
+
const observed = this.observePeerSyncCapabilities({
|
|
19315
19423
|
peerHash: receiveFromHash,
|
|
19316
19424
|
capabilities: msg.capabilities,
|
|
19317
19425
|
transportSession: capabilityTransportSession,
|
|
19318
19426
|
timestamp: capabilityTimestamp,
|
|
19319
|
-
})
|
|
19320
|
-
receiveSession?.phase === "open"
|
|
19321
|
-
|
|
19322
|
-
|
|
19323
|
-
|
|
19324
|
-
|
|
19325
|
-
)
|
|
19427
|
+
});
|
|
19428
|
+
if (observed && receiveSession?.phase === "open") {
|
|
19429
|
+
this.promoteReplicationInfoV2ReceiveCapability(
|
|
19430
|
+
context.from,
|
|
19431
|
+
receiveSession,
|
|
19432
|
+
);
|
|
19433
|
+
} else if (observed && receiveSession === null) {
|
|
19434
|
+
// A capability can arrive before the sender's topic Subscribe after
|
|
19435
|
+
// reconnect. Ask that authenticated peer for its authoritative
|
|
19436
|
+
// subscriber snapshot; the resulting Subscribe creates the real
|
|
19437
|
+
// PeerSession and completes the symmetric capability handshake.
|
|
19438
|
+
// Never synthesize membership from capability traffic alone.
|
|
19439
|
+
this.requestSubscriberSnapshotForCapability(context.from);
|
|
19440
|
+
}
|
|
19326
19441
|
}
|
|
19327
19442
|
}
|
|
19328
19443
|
return;
|
|
@@ -19889,7 +20004,13 @@ export class SharedLog<
|
|
|
19889
20004
|
return;
|
|
19890
20005
|
}
|
|
19891
20006
|
this._liveness.markReplicatorActivity(fromHash);
|
|
19892
|
-
|
|
20007
|
+
// A committed V2 announcement is applied progress: the peer answers,
|
|
20008
|
+
// so recovery re-solicitation may restart from the base interval.
|
|
20009
|
+
this.resetReplicationInfoV2RecoveryEscalation(fromHash);
|
|
20010
|
+
if (
|
|
20011
|
+
msg instanceof FullReplicationInfoV2Message &&
|
|
20012
|
+
this.legacyReplicationInfoEnabled
|
|
20013
|
+
) {
|
|
19893
20014
|
this.cancelReplicationInfoRequests(fromHash);
|
|
19894
20015
|
}
|
|
19895
20016
|
});
|
|
@@ -20553,6 +20674,7 @@ export class SharedLog<
|
|
|
20553
20674
|
}, timeoutMs);
|
|
20554
20675
|
|
|
20555
20676
|
let requestAttempts = 0;
|
|
20677
|
+
let subscriberSnapshotInFlight: Promise<void> | undefined;
|
|
20556
20678
|
const requestIntervalMs = this.waitForReplicatorRequestIntervalMs;
|
|
20557
20679
|
const maxRequestAttempts =
|
|
20558
20680
|
this.waitForReplicatorRequestMaxAttempts ??
|
|
@@ -20560,6 +20682,26 @@ export class SharedLog<
|
|
|
20560
20682
|
WAIT_FOR_REPLICATOR_REQUEST_MIN_ATTEMPTS,
|
|
20561
20683
|
Math.ceil(timeoutMs / requestIntervalMs),
|
|
20562
20684
|
);
|
|
20685
|
+
const requestSubscriberSnapshot = () => {
|
|
20686
|
+
if (subscriberSnapshotInFlight) {
|
|
20687
|
+
return;
|
|
20688
|
+
}
|
|
20689
|
+
subscriberSnapshotInFlight = Promise.resolve()
|
|
20690
|
+
.then(async () => {
|
|
20691
|
+
if (settled || this.closed) {
|
|
20692
|
+
return;
|
|
20693
|
+
}
|
|
20694
|
+
await this.node.services.pubsub.requestSubscribers(this.topic, key);
|
|
20695
|
+
})
|
|
20696
|
+
.catch((error) => {
|
|
20697
|
+
if (!isNotStartedError(error as Error)) {
|
|
20698
|
+
logger.error(error?.toString?.() ?? String(error));
|
|
20699
|
+
}
|
|
20700
|
+
})
|
|
20701
|
+
.finally(() => {
|
|
20702
|
+
subscriberSnapshotInFlight = undefined;
|
|
20703
|
+
});
|
|
20704
|
+
};
|
|
20563
20705
|
|
|
20564
20706
|
const requestReplicationInfo = () => {
|
|
20565
20707
|
if (settled || this.closed) {
|
|
@@ -20572,17 +20714,35 @@ export class SharedLog<
|
|
|
20572
20714
|
|
|
20573
20715
|
requestAttempts++;
|
|
20574
20716
|
|
|
20575
|
-
this.
|
|
20576
|
-
.
|
|
20577
|
-
|
|
20578
|
-
|
|
20579
|
-
|
|
20580
|
-
|
|
20581
|
-
|
|
20582
|
-
|
|
20583
|
-
|
|
20584
|
-
|
|
20585
|
-
|
|
20717
|
+
if (this.legacyReplicationInfoEnabled) {
|
|
20718
|
+
this.rpc
|
|
20719
|
+
.send(new RequestReplicationInfoMessage(), {
|
|
20720
|
+
mode: new AcknowledgeDelivery({ redundancy: 1, to: [key] }),
|
|
20721
|
+
})
|
|
20722
|
+
.catch((e) => {
|
|
20723
|
+
// Best-effort: missing peers / unopened RPC should not fail the wait logic.
|
|
20724
|
+
if (isNotStartedError(e as Error)) {
|
|
20725
|
+
return;
|
|
20726
|
+
}
|
|
20727
|
+
logger.error(e?.toString?.() ?? String(e));
|
|
20728
|
+
});
|
|
20729
|
+
} else {
|
|
20730
|
+
const peerHash = key.hashcode();
|
|
20731
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
20732
|
+
if (peerSession?.phase === "open") {
|
|
20733
|
+
this._v2Receive.resumeParkedRequest({
|
|
20734
|
+
peerHash,
|
|
20735
|
+
peerSession,
|
|
20736
|
+
receiveEpoch: this._peerSessions.receiveEpoch(peerHash),
|
|
20737
|
+
});
|
|
20738
|
+
} else if (peerSession === null || peerSession.phase === "departing") {
|
|
20739
|
+
// A peer can be known to routing before its SharedLog topic
|
|
20740
|
+
// subscription has been observed. Legacy requests used to bootstrap
|
|
20741
|
+
// that case directly; V2 needs an authoritative Subscribe snapshot
|
|
20742
|
+
// before it can create a fenced PeerSession and request a Full.
|
|
20743
|
+
requestSubscriberSnapshot();
|
|
20744
|
+
}
|
|
20745
|
+
}
|
|
20586
20746
|
|
|
20587
20747
|
if (requestAttempts < maxRequestAttempts) {
|
|
20588
20748
|
requestTimer = setTimeout(requestReplicationInfo, requestIntervalMs);
|
|
@@ -23316,6 +23476,132 @@ export class SharedLog<
|
|
|
23316
23476
|
this._replicationInfoRequestByPeer.delete(peerHash);
|
|
23317
23477
|
}
|
|
23318
23478
|
|
|
23479
|
+
/**
|
|
23480
|
+
* Applied V2 progress from a peer (a committed Full/Added/Stopped, or a
|
|
23481
|
+
* rotated capability generation) proves the peer answers. Reset the
|
|
23482
|
+
* recovery scheduler's unpark escalation so a later stall restarts from
|
|
23483
|
+
* the base interval. Peer-session rotation resets implicitly: the recovery
|
|
23484
|
+
* scheduler creates a fresh per-session state.
|
|
23485
|
+
*/
|
|
23486
|
+
private resetReplicationInfoV2RecoveryEscalation(peerHash: string) {
|
|
23487
|
+
const state = this._replicationInfoRequestByPeer.get(peerHash);
|
|
23488
|
+
if (!state || state.peerSession === undefined) {
|
|
23489
|
+
return;
|
|
23490
|
+
}
|
|
23491
|
+
state.attempts = 0;
|
|
23492
|
+
state.parkedSinceMs = undefined;
|
|
23493
|
+
}
|
|
23494
|
+
|
|
23495
|
+
private scheduleReplicationInfoV2Recovery(
|
|
23496
|
+
peer: PublicSignKey,
|
|
23497
|
+
replicationLifecycleController = this._instanceLifecycle
|
|
23498
|
+
?.membershipLifecycleController,
|
|
23499
|
+
) {
|
|
23500
|
+
if (
|
|
23501
|
+
!replicationLifecycleController ||
|
|
23502
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
23503
|
+
) {
|
|
23504
|
+
return;
|
|
23505
|
+
}
|
|
23506
|
+
const peerHash = peer.hashcode();
|
|
23507
|
+
const peerSession = this._peerSessions.current(peerHash);
|
|
23508
|
+
if (!peerSession || peerSession.phase !== "open") {
|
|
23509
|
+
return;
|
|
23510
|
+
}
|
|
23511
|
+
const requestStates = this._replicationInfoRequestByPeer;
|
|
23512
|
+
const existing = requestStates.get(peerHash);
|
|
23513
|
+
if (existing?.peerSession === peerSession) {
|
|
23514
|
+
return;
|
|
23515
|
+
}
|
|
23516
|
+
if (existing) {
|
|
23517
|
+
if (existing.timer) {
|
|
23518
|
+
clearTimeout(existing.timer);
|
|
23519
|
+
}
|
|
23520
|
+
requestStates.delete(peerHash);
|
|
23521
|
+
}
|
|
23522
|
+
const state: {
|
|
23523
|
+
attempts: number;
|
|
23524
|
+
timer?: ReturnType<typeof setTimeout>;
|
|
23525
|
+
peerSession: PeerSession;
|
|
23526
|
+
parkedSinceMs?: number;
|
|
23527
|
+
} = {
|
|
23528
|
+
attempts: 0,
|
|
23529
|
+
peerSession,
|
|
23530
|
+
};
|
|
23531
|
+
requestStates.set(peerHash, state);
|
|
23532
|
+
const cancel = () => {
|
|
23533
|
+
if (requestStates.get(peerHash) !== state) {
|
|
23534
|
+
return;
|
|
23535
|
+
}
|
|
23536
|
+
if (state.timer) {
|
|
23537
|
+
clearTimeout(state.timer);
|
|
23538
|
+
}
|
|
23539
|
+
requestStates.delete(peerHash);
|
|
23540
|
+
};
|
|
23541
|
+
const intervalMs = Math.max(50, this.waitForReplicatorRequestIntervalMs);
|
|
23542
|
+
const maxUnparkDelayMs = Math.max(
|
|
23543
|
+
intervalMs,
|
|
23544
|
+
REPLICATION_INFO_V2_RECOVERY_MAX_UNPARK_DELAY,
|
|
23545
|
+
);
|
|
23546
|
+
const unparkDelayMs = () =>
|
|
23547
|
+
Math.min(
|
|
23548
|
+
maxUnparkDelayMs,
|
|
23549
|
+
intervalMs *
|
|
23550
|
+
2 **
|
|
23551
|
+
Math.min(
|
|
23552
|
+
state.attempts,
|
|
23553
|
+
REPLICATION_INFO_V2_RECOVERY_MAX_UNPARK_EXPONENT,
|
|
23554
|
+
),
|
|
23555
|
+
);
|
|
23556
|
+
const arm = (delayMs: number) => {
|
|
23557
|
+
state.timer = setTimeout(tick, delayMs);
|
|
23558
|
+
state.timer.unref?.();
|
|
23559
|
+
};
|
|
23560
|
+
const tick = () => {
|
|
23561
|
+
if (
|
|
23562
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
23563
|
+
peerSession.phase !== "open" ||
|
|
23564
|
+
!this._peerSessions.isCurrent(peerHash, peerSession)
|
|
23565
|
+
) {
|
|
23566
|
+
cancel();
|
|
23567
|
+
return;
|
|
23568
|
+
}
|
|
23569
|
+
const receiveEpoch = this._peerSessions.receiveEpoch(peerHash);
|
|
23570
|
+
if (
|
|
23571
|
+
!this._v2Receive.isRequestParked({ peerHash, peerSession, receiveEpoch })
|
|
23572
|
+
) {
|
|
23573
|
+
// Active, or a bounded request cycle is still running its own
|
|
23574
|
+
// exponential retries. Keep polling for the next park.
|
|
23575
|
+
state.parkedSinceMs = undefined;
|
|
23576
|
+
arm(intervalMs);
|
|
23577
|
+
return;
|
|
23578
|
+
}
|
|
23579
|
+
const now = Date.now();
|
|
23580
|
+
if (state.parkedSinceMs === undefined) {
|
|
23581
|
+
state.parkedSinceMs = now;
|
|
23582
|
+
}
|
|
23583
|
+
const resumeAtMs = state.parkedSinceMs + unparkDelayMs();
|
|
23584
|
+
if (now < resumeAtMs) {
|
|
23585
|
+
arm(Math.max(50, resumeAtMs - now));
|
|
23586
|
+
return;
|
|
23587
|
+
}
|
|
23588
|
+
if (
|
|
23589
|
+
this._v2Receive.resumeParkedRequest({
|
|
23590
|
+
peerHash,
|
|
23591
|
+
peerSession,
|
|
23592
|
+
receiveEpoch,
|
|
23593
|
+
})
|
|
23594
|
+
) {
|
|
23595
|
+
// Fruitless until proven otherwise: escalate the next unpark wait.
|
|
23596
|
+
// Applied progress resets via resetReplicationInfoV2RecoveryEscalation.
|
|
23597
|
+
state.attempts++;
|
|
23598
|
+
state.parkedSinceMs = undefined;
|
|
23599
|
+
}
|
|
23600
|
+
arm(intervalMs);
|
|
23601
|
+
};
|
|
23602
|
+
tick();
|
|
23603
|
+
}
|
|
23604
|
+
|
|
23319
23605
|
private scheduleReplicationInfoRequests(
|
|
23320
23606
|
peer: PublicSignKey,
|
|
23321
23607
|
replicationLifecycleController = this._instanceLifecycle
|
|
@@ -23327,6 +23613,13 @@ export class SharedLog<
|
|
|
23327
23613
|
) {
|
|
23328
23614
|
return;
|
|
23329
23615
|
}
|
|
23616
|
+
if (!this.legacyReplicationInfoEnabled) {
|
|
23617
|
+
this.scheduleReplicationInfoV2Recovery(
|
|
23618
|
+
peer,
|
|
23619
|
+
replicationLifecycleController,
|
|
23620
|
+
);
|
|
23621
|
+
return;
|
|
23622
|
+
}
|
|
23330
23623
|
const peerHash = peer.hashcode();
|
|
23331
23624
|
const requestStates = this._replicationInfoRequestByPeer;
|
|
23332
23625
|
if (requestStates.has(peerHash)) {
|
|
@@ -23360,7 +23653,6 @@ export class SharedLog<
|
|
|
23360
23653
|
cancel();
|
|
23361
23654
|
return;
|
|
23362
23655
|
}
|
|
23363
|
-
|
|
23364
23656
|
state.attempts++;
|
|
23365
23657
|
|
|
23366
23658
|
this.rpc
|
|
@@ -23397,6 +23689,7 @@ export class SharedLog<
|
|
|
23397
23689
|
topics: string[],
|
|
23398
23690
|
subscribed: boolean,
|
|
23399
23691
|
subscriptionEpoch?: PeerSession,
|
|
23692
|
+
subscriptionTransportSession?: bigint,
|
|
23400
23693
|
) {
|
|
23401
23694
|
if (!topics.includes(this.topic)) {
|
|
23402
23695
|
return;
|
|
@@ -23419,13 +23712,31 @@ export class SharedLog<
|
|
|
23419
23712
|
if (!ownsSubscriptionEpoch()) {
|
|
23420
23713
|
return;
|
|
23421
23714
|
}
|
|
23715
|
+
// A reconnect can arrive before the previous exact-session recovery tick
|
|
23716
|
+
// observes its stale session. Retire that job synchronously so it cannot
|
|
23717
|
+
// suppress the replacement session's scheduler in the shared peer slot.
|
|
23718
|
+
this.cancelReplicationInfoRequests(peerHash);
|
|
23422
23719
|
// A destination stream is scoped to exactly one topic-subscription
|
|
23423
23720
|
// session. Abort the predecessor synchronously before either barrier can
|
|
23424
23721
|
// yield; a late queue completion must never enter the new session.
|
|
23425
23722
|
this._v2Receive.clearPeer(peerHash);
|
|
23426
23723
|
this._v2Send.clearPeer(peerHash);
|
|
23427
|
-
|
|
23428
|
-
|
|
23724
|
+
const capabilityTransportSession =
|
|
23725
|
+
this._peerSyncCapabilitySessions.get(peerHash);
|
|
23726
|
+
const canInheritCapability =
|
|
23727
|
+
subscribed &&
|
|
23728
|
+
(subscriptionTransportSession !== undefined
|
|
23729
|
+
? capabilityTransportSession === subscriptionTransportSession
|
|
23730
|
+
: !expectedSubscriptionEpoch.hasPredecessor);
|
|
23731
|
+
if (!canInheritCapability) {
|
|
23732
|
+
// Departures and successor openings revoke the signed transport binding.
|
|
23733
|
+
// A live successor may inherit a capability that arrived just before its
|
|
23734
|
+
// Subscribe only when both signed frames belong to the same transport
|
|
23735
|
+
// generation. Snapshot fallbacks lack that proof, so only their first
|
|
23736
|
+
// opening may inherit pre-opening capability state.
|
|
23737
|
+
this._peerSyncCapabilitySessions.delete(peerHash);
|
|
23738
|
+
this._peerSyncCapabilityTimestamps.delete(peerHash);
|
|
23739
|
+
}
|
|
23429
23740
|
if (subscribed) {
|
|
23430
23741
|
const pendingOpeningCapabilities =
|
|
23431
23742
|
this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
@@ -23577,6 +23888,17 @@ export class SharedLog<
|
|
|
23577
23888
|
receiveEpoch,
|
|
23578
23889
|
signal: replicationLifecycleController.signal,
|
|
23579
23890
|
});
|
|
23891
|
+
if (!this.legacyReplicationInfoEnabled) {
|
|
23892
|
+
// Current logs have no legacy startup work to order ahead of RequestV2.
|
|
23893
|
+
// Releasing is synchronous and exact-session fenced; the ACK may still
|
|
23894
|
+
// arrive later and promote readiness through the coordinator.
|
|
23895
|
+
localCapabilityAdvertisement.releaseLegacyBarrier();
|
|
23896
|
+
this.scheduleReplicationInfoV2Recovery(
|
|
23897
|
+
publicKey,
|
|
23898
|
+
replicationLifecycleController,
|
|
23899
|
+
);
|
|
23900
|
+
return;
|
|
23901
|
+
}
|
|
23580
23902
|
|
|
23581
23903
|
try {
|
|
23582
23904
|
let replicationSegments: ReplicationRangeIndexable<R>[];
|
|
@@ -25105,6 +25427,7 @@ export class SharedLog<
|
|
|
25105
25427
|
evt.detail.topics,
|
|
25106
25428
|
true,
|
|
25107
25429
|
subscriptionEpoch,
|
|
25430
|
+
evt.detail.session,
|
|
25108
25431
|
);
|
|
25109
25432
|
}
|
|
25110
25433
|
|
package/src/peer-session.ts
CHANGED
|
@@ -31,6 +31,11 @@ export type PeerReceiveAdmissionOptions = {
|
|
|
31
31
|
export class PeerSession {
|
|
32
32
|
readonly peerHash: string;
|
|
33
33
|
readonly kind: PeerSessionKind;
|
|
34
|
+
// True when rotate() superseded an earlier subscription generation. This is
|
|
35
|
+
// deliberately independent of the predecessor's phase/kind: a newer
|
|
36
|
+
// transport can announce a subscription without first delivering an
|
|
37
|
+
// unsubscribe, and must not inherit the old transport's signed capability.
|
|
38
|
+
readonly hasPredecessor: boolean;
|
|
34
39
|
// The lifecycle controller live at rotation. All current seams pair the
|
|
35
40
|
// epoch check with a lifecycle check against a controller captured in the
|
|
36
41
|
// same synchronous window as the epoch advance; capturing it here
|
|
@@ -59,10 +64,12 @@ export class PeerSession {
|
|
|
59
64
|
peerHash: string,
|
|
60
65
|
kind: PeerSessionKind,
|
|
61
66
|
replicationLifecycleController: AbortController | undefined,
|
|
67
|
+
hasPredecessor: boolean,
|
|
62
68
|
) {
|
|
63
69
|
this.peerHash = peerHash;
|
|
64
70
|
this.kind = kind;
|
|
65
71
|
this.replicationLifecycleController = replicationLifecycleController;
|
|
72
|
+
this.hasPredecessor = hasPredecessor;
|
|
66
73
|
this.phase = kind;
|
|
67
74
|
}
|
|
68
75
|
|
|
@@ -278,6 +285,7 @@ export class PeerSessionRegistry {
|
|
|
278
285
|
peerHash,
|
|
279
286
|
kind,
|
|
280
287
|
this.deps.getReplicationLifecycleController(),
|
|
288
|
+
previous !== undefined,
|
|
281
289
|
);
|
|
282
290
|
this.sessions.set(peerHash, next);
|
|
283
291
|
return next;
|