@peerbit/shared-log 13.2.9 → 13.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +90 -9
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1643 -203
- package/dist/src/index.js.map +1 -1
- package/package.json +13 -13
- package/src/index.ts +2316 -281
package/src/index.ts
CHANGED
|
@@ -6,7 +6,11 @@ import {
|
|
|
6
6
|
variant,
|
|
7
7
|
} from "@dao-xyz/borsh";
|
|
8
8
|
import type { AnyStore } from "@peerbit/any-store";
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
AnyBlockStore,
|
|
11
|
+
type EagerBlocksSetting,
|
|
12
|
+
RemoteBlocks,
|
|
13
|
+
} from "@peerbit/blocks";
|
|
10
14
|
import { cidifyString } from "@peerbit/blocks-interface";
|
|
11
15
|
import { Cache } from "@peerbit/cache";
|
|
12
16
|
import {
|
|
@@ -105,6 +109,7 @@ import {
|
|
|
105
109
|
BACKGROUND_MESSAGE_PRIORITY,
|
|
106
110
|
CONVERGENCE_MESSAGE_PRIORITY,
|
|
107
111
|
DataMessage,
|
|
112
|
+
DeliveryError,
|
|
108
113
|
MessageHeader,
|
|
109
114
|
NotStartedError,
|
|
110
115
|
type RouteHint,
|
|
@@ -278,10 +283,20 @@ type PendingIHave<T> = {
|
|
|
278
283
|
resetTimeout: () => void;
|
|
279
284
|
requesting: Set<string>;
|
|
280
285
|
clear: () => void;
|
|
281
|
-
callback: (entry: Entry<T>) => void
|
|
286
|
+
callback: (entry: Entry<T>) => MaybePromise<void>;
|
|
282
287
|
expiresAt?: number;
|
|
283
288
|
};
|
|
284
289
|
|
|
290
|
+
type PeerReceiveLeaseBucket = {
|
|
291
|
+
active: number;
|
|
292
|
+
drain?: DeferredPromise<void>;
|
|
293
|
+
};
|
|
294
|
+
|
|
295
|
+
type PeerReceiveLeaseState = {
|
|
296
|
+
current: PeerReceiveLeaseBucket;
|
|
297
|
+
activeBuckets: Set<PeerReceiveLeaseBucket>;
|
|
298
|
+
};
|
|
299
|
+
|
|
285
300
|
const toLocalPublicSignKey = (
|
|
286
301
|
key: PublicSignKey | string,
|
|
287
302
|
): PublicSignKey | undefined => {
|
|
@@ -2307,6 +2322,64 @@ const isTransientReplicationAnnouncementError = (
|
|
|
2307
2322
|
return constructorName === "TimeoutError" || name === "TimeoutError";
|
|
2308
2323
|
};
|
|
2309
2324
|
|
|
2325
|
+
/**
|
|
2326
|
+
* Directed transport-delivery repair is allowed to retry explicit delivery
|
|
2327
|
+
* failures in addition to timeouts. A DirectStream ACK confirms receipt of the
|
|
2328
|
+
* signed envelope, not successful application by the receiver. Keep this
|
|
2329
|
+
* separate from the primary fanout classifier above so replicate() rejection
|
|
2330
|
+
* semantics remain unchanged for programming, serialization, and lifecycle
|
|
2331
|
+
* errors.
|
|
2332
|
+
*/
|
|
2333
|
+
const isTransientReplicationAnnouncementRepairError = (
|
|
2334
|
+
error: unknown,
|
|
2335
|
+
seen = new Set<unknown>(),
|
|
2336
|
+
): boolean => {
|
|
2337
|
+
if (
|
|
2338
|
+
error != null &&
|
|
2339
|
+
(typeof error === "object" || typeof error === "function")
|
|
2340
|
+
) {
|
|
2341
|
+
if (seen.has(error)) {
|
|
2342
|
+
return false;
|
|
2343
|
+
}
|
|
2344
|
+
seen.add(error);
|
|
2345
|
+
}
|
|
2346
|
+
|
|
2347
|
+
if (error instanceof DeliveryError || error instanceof TimeoutError) {
|
|
2348
|
+
return true;
|
|
2349
|
+
}
|
|
2350
|
+
|
|
2351
|
+
const nested = (error as { errors?: unknown })?.errors;
|
|
2352
|
+
if (Array.isArray(nested) && nested.length > 0) {
|
|
2353
|
+
return nested.every((item) =>
|
|
2354
|
+
isTransientReplicationAnnouncementRepairError(item, new Set(seen)),
|
|
2355
|
+
);
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
const cause = (error as { cause?: unknown })?.cause;
|
|
2359
|
+
if (
|
|
2360
|
+
cause != null &&
|
|
2361
|
+
isTransientReplicationAnnouncementRepairError(cause, seen)
|
|
2362
|
+
) {
|
|
2363
|
+
return true;
|
|
2364
|
+
}
|
|
2365
|
+
|
|
2366
|
+
const constructorName =
|
|
2367
|
+
typeof (error as { constructor?: { name?: unknown } })?.constructor
|
|
2368
|
+
?.name === "string"
|
|
2369
|
+
? (error as { constructor: { name: string } }).constructor.name
|
|
2370
|
+
: "";
|
|
2371
|
+
const name =
|
|
2372
|
+
typeof (error as { name?: unknown })?.name === "string"
|
|
2373
|
+
? (error as { name: string }).name
|
|
2374
|
+
: "";
|
|
2375
|
+
return (
|
|
2376
|
+
constructorName === "DeliveryError" ||
|
|
2377
|
+
name === "DeliveryError" ||
|
|
2378
|
+
constructorName === "TimeoutError" ||
|
|
2379
|
+
name === "TimeoutError"
|
|
2380
|
+
);
|
|
2381
|
+
};
|
|
2382
|
+
|
|
2310
2383
|
interface IndexableDomain<R extends "u32" | "u64"> {
|
|
2311
2384
|
numbers: Numbers<R>;
|
|
2312
2385
|
constructorEntry: new (properties: {
|
|
@@ -2511,32 +2584,61 @@ export type SharedLogOptions<
|
|
|
2511
2584
|
strictFullReplicaFallback?: boolean;
|
|
2512
2585
|
compatibility?: number;
|
|
2513
2586
|
domain?: ReplicationDomainConstructor<D>;
|
|
2514
|
-
eagerBlocks?:
|
|
2587
|
+
eagerBlocks?: EagerBlocksSetting;
|
|
2515
2588
|
fanout?: SharedLogFanoutOptions;
|
|
2516
2589
|
};
|
|
2517
2590
|
|
|
2518
2591
|
/**
|
|
2519
|
-
*
|
|
2520
|
-
* it
|
|
2521
|
-
*
|
|
2522
|
-
* explicit per-open options (including `false`) always win.
|
|
2523
|
-
* property on the client, behavior is unchanged.
|
|
2592
|
+
* Runtime defaults a client can advertise for shared-log programs opened on
|
|
2593
|
+
* it. The historical name is retained because the peerbit native network
|
|
2594
|
+
* preset introduced this hook. Defaults fill in open options the caller left
|
|
2595
|
+
* undefined; explicit per-open options (including `false`) always win.
|
|
2596
|
+
* Without the property on the client, behavior is unchanged.
|
|
2524
2597
|
*/
|
|
2525
2598
|
export type SharedLogNativeDefaults = {
|
|
2526
2599
|
nativeBackbone?: SharedLogOptions<any, any, any>["nativeBackbone"];
|
|
2527
2600
|
nativeGraph?: LogProperties<any>["nativeGraph"];
|
|
2528
2601
|
sync?: Pick<SyncOptions<any>, "rawExchangeHeads" | "nativeWireSync">;
|
|
2602
|
+
/**
|
|
2603
|
+
* Per-channel defaults applied only when the caller opts into SharedLog
|
|
2604
|
+
* fanout. Explicit per-open channel options take precedence.
|
|
2605
|
+
*/
|
|
2606
|
+
fanout?: Pick<SharedLogFanoutOptions, "channel">;
|
|
2529
2607
|
};
|
|
2530
2608
|
|
|
2531
2609
|
type NodeWithSharedLogNativeDefaults = {
|
|
2532
2610
|
sharedLogNativeDefaults?: SharedLogNativeDefaults;
|
|
2533
2611
|
};
|
|
2534
2612
|
|
|
2613
|
+
type SharedLogFanoutChannelOptions = NonNullable<
|
|
2614
|
+
SharedLogFanoutOptions["channel"]
|
|
2615
|
+
>;
|
|
2616
|
+
|
|
2617
|
+
const mergeDefinedFanoutChannelOptions = (
|
|
2618
|
+
...sources: Array<SharedLogFanoutChannelOptions | undefined>
|
|
2619
|
+
): SharedLogFanoutChannelOptions | undefined => {
|
|
2620
|
+
let merged: Record<string, unknown> | undefined;
|
|
2621
|
+
for (const source of sources) {
|
|
2622
|
+
if (!source) {
|
|
2623
|
+
continue;
|
|
2624
|
+
}
|
|
2625
|
+
for (const [key, value] of Object.entries(source)) {
|
|
2626
|
+
if (value === undefined) {
|
|
2627
|
+
continue;
|
|
2628
|
+
}
|
|
2629
|
+
merged ??= {};
|
|
2630
|
+
merged[key] = value;
|
|
2631
|
+
}
|
|
2632
|
+
}
|
|
2633
|
+
return merged as SharedLogFanoutChannelOptions | undefined;
|
|
2634
|
+
};
|
|
2635
|
+
|
|
2535
2636
|
const applySharedLogNativeDefaults = <
|
|
2536
2637
|
O extends {
|
|
2537
2638
|
nativeBackbone?: SharedLogOptions<any, any, any>["nativeBackbone"];
|
|
2538
2639
|
nativeGraph?: LogProperties<any>["nativeGraph"];
|
|
2539
2640
|
sync?: SyncOptions<any>;
|
|
2641
|
+
fanout?: SharedLogFanoutOptions;
|
|
2540
2642
|
},
|
|
2541
2643
|
>(
|
|
2542
2644
|
options: O | undefined,
|
|
@@ -2555,11 +2657,21 @@ const applySharedLogNativeDefaults = <
|
|
|
2555
2657
|
options?.sync?.nativeWireSync ?? defaults.sync?.nativeWireSync,
|
|
2556
2658
|
}
|
|
2557
2659
|
: undefined;
|
|
2660
|
+
const fanout = options?.fanout
|
|
2661
|
+
? {
|
|
2662
|
+
...options.fanout,
|
|
2663
|
+
channel: mergeDefinedFanoutChannelOptions(
|
|
2664
|
+
defaults.fanout?.channel,
|
|
2665
|
+
options.fanout.channel,
|
|
2666
|
+
),
|
|
2667
|
+
}
|
|
2668
|
+
: undefined;
|
|
2558
2669
|
return {
|
|
2559
2670
|
...options,
|
|
2560
2671
|
nativeBackbone: options?.nativeBackbone ?? defaults.nativeBackbone,
|
|
2561
2672
|
nativeGraph: options?.nativeGraph ?? defaults.nativeGraph,
|
|
2562
2673
|
sync,
|
|
2674
|
+
fanout,
|
|
2563
2675
|
} as O;
|
|
2564
2676
|
};
|
|
2565
2677
|
|
|
@@ -2582,6 +2694,14 @@ const CHECKED_PRUNE_RETRY_MAX_DELAY_MS = 30_000;
|
|
|
2582
2694
|
// DONT SET THIS ANY LOWER, because it will make the pid controller unstable as the system responses are not fast enough to updates from the pid controller
|
|
2583
2695
|
const RECALCULATE_PARTICIPATION_DEBOUNCE_INTERVAL = 1000;
|
|
2584
2696
|
const REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL = 1000;
|
|
2697
|
+
const REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL = 1000;
|
|
2698
|
+
const REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS = 3;
|
|
2699
|
+
// Repair one bounded cohort per mutation generation. The subscriber snapshot
|
|
2700
|
+
// is a best-effort cache and can contain thousands of entries, so attempting
|
|
2701
|
+
// the whole cache after every role mutation would turn convergence repair into
|
|
2702
|
+
// an unbounded burst of separately signed, acknowledged messages. A cursor
|
|
2703
|
+
// retained across generations rotates best-effort coverage over later changes.
|
|
2704
|
+
const REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION = 8;
|
|
2585
2705
|
const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE = 0.01;
|
|
2586
2706
|
const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_CPU_LIMIT = 0.005;
|
|
2587
2707
|
const RECALCULATE_PARTICIPATION_MIN_RELATIVE_CHANGE_WITH_MEMORY_LIMIT = 0.001;
|
|
@@ -2618,6 +2738,13 @@ const NATIVE_ED25519_VERIFY_BATCH_MIN_ENTRIES = 16;
|
|
|
2618
2738
|
const hasPreverifiedSignature = (entry: Entry<any>) =>
|
|
2619
2739
|
(entry as { __peerbitSignatureVerified?: unknown })
|
|
2620
2740
|
.__peerbitSignatureVerified === true;
|
|
2741
|
+
|
|
2742
|
+
type ReplicationAnnouncementRepairTarget = {
|
|
2743
|
+
key: PublicSignKey;
|
|
2744
|
+
generation: number;
|
|
2745
|
+
attempts: number;
|
|
2746
|
+
done: boolean;
|
|
2747
|
+
};
|
|
2621
2748
|
// In sparse topologies (browser/relay), peers can learn about replicators via broadcast
|
|
2622
2749
|
// replication announcements without having a direct connection that emits unsubscribe
|
|
2623
2750
|
// on abrupt churn. Probe conservatively so a single missed ACK does not evict a
|
|
@@ -2631,9 +2758,13 @@ const REPLICATOR_LIVENESS_PROBE_FAILURES_TO_EVICT = 2;
|
|
|
2631
2758
|
const CHURN_REPAIR_RETRY_SCHEDULE_MS = [
|
|
2632
2759
|
0, 1_000, 3_000, 7_000, 15_000, 30_000, 45_000,
|
|
2633
2760
|
];
|
|
2761
|
+
// Preserve the bounded retry window for transient local misses, but serialize
|
|
2762
|
+
// delayed warmup sends per target so fixed snapshots cannot overlap and amplify
|
|
2763
|
+
// large transfers. Every queued pass re-checks current peer knowledge on entry.
|
|
2634
2764
|
const JOIN_WARMUP_RETRY_SCHEDULE_MS = [
|
|
2635
2765
|
0, 1_000, 3_000, 7_000, 15_000, 30_000, 60_000,
|
|
2636
2766
|
];
|
|
2767
|
+
const JOIN_WARMUP_SEND_SPACING_MS = 250;
|
|
2637
2768
|
const JOIN_AUTHORITATIVE_RETRY_SCHEDULE_MS = [
|
|
2638
2769
|
0, 1_000, 3_000, 7_000, 15_000, 30_000, 60_000,
|
|
2639
2770
|
];
|
|
@@ -2664,6 +2795,48 @@ type RepairMetricBucket = {
|
|
|
2664
2795
|
};
|
|
2665
2796
|
type RepairMetrics = Record<RepairDispatchMode, RepairMetricBucket>;
|
|
2666
2797
|
|
|
2798
|
+
type JoinWarmupSendState<R extends "u32" | "u64"> = {
|
|
2799
|
+
bypassKnownPeerHints: boolean;
|
|
2800
|
+
entries: Map<string, RepairDispatchEntry<R>>;
|
|
2801
|
+
generation: object;
|
|
2802
|
+
lastCompletedAt: number;
|
|
2803
|
+
pending: boolean;
|
|
2804
|
+
running: boolean;
|
|
2805
|
+
};
|
|
2806
|
+
|
|
2807
|
+
type JoinWarmupRetryTimer = {
|
|
2808
|
+
handle: ReturnType<typeof setTimeout>;
|
|
2809
|
+
resolve?: () => void;
|
|
2810
|
+
};
|
|
2811
|
+
|
|
2812
|
+
type JoinWarmupScheduledRetryBatch<R extends "u32" | "u64"> = {
|
|
2813
|
+
bypassKnownPeerHints: boolean;
|
|
2814
|
+
entries: Map<string, RepairDispatchEntry<R>>;
|
|
2815
|
+
remainingAttempts: number;
|
|
2816
|
+
};
|
|
2817
|
+
|
|
2818
|
+
type JoinWarmupScheduledRetryCohort<R extends "u32" | "u64"> = {
|
|
2819
|
+
batches: JoinWarmupScheduledRetryBatch<R>[];
|
|
2820
|
+
dueAt: number;
|
|
2821
|
+
};
|
|
2822
|
+
|
|
2823
|
+
type JoinWarmupScheduledRetrySlot<R extends "u32" | "u64"> = {
|
|
2824
|
+
cohorts: JoinWarmupScheduledRetryCohort<R>[];
|
|
2825
|
+
head: number;
|
|
2826
|
+
timer?: JoinWarmupRetryTimer;
|
|
2827
|
+
timerDueAt?: number;
|
|
2828
|
+
};
|
|
2829
|
+
|
|
2830
|
+
type JoinWarmupScheduledRetries<R extends "u32" | "u64"> = {
|
|
2831
|
+
generation: object;
|
|
2832
|
+
slotsByDelay: Map<number, JoinWarmupScheduledRetrySlot<R>>;
|
|
2833
|
+
};
|
|
2834
|
+
|
|
2835
|
+
type RepairSweepOptimisticPeerState = {
|
|
2836
|
+
count: number;
|
|
2837
|
+
generation: object;
|
|
2838
|
+
};
|
|
2839
|
+
|
|
2667
2840
|
const REPAIR_DISPATCH_MODES: RepairDispatchMode[] = [
|
|
2668
2841
|
"join-warmup",
|
|
2669
2842
|
"join-authoritative",
|
|
@@ -2816,6 +2989,11 @@ export type DeliveryOptions = {
|
|
|
2816
2989
|
reliability?: DeliveryReliability;
|
|
2817
2990
|
minAcks?: number;
|
|
2818
2991
|
requireRecipients?: boolean;
|
|
2992
|
+
/**
|
|
2993
|
+
* Transport priority for directed RPC delivery. Fanout unicast already uses
|
|
2994
|
+
* its control lane, so this only changes the direct/fallback RPC path.
|
|
2995
|
+
*/
|
|
2996
|
+
priority?: number;
|
|
2819
2997
|
timeout?: number;
|
|
2820
2998
|
signal?: AbortSignal;
|
|
2821
2999
|
};
|
|
@@ -3094,6 +3272,13 @@ export interface SharedLogEvents extends ProgramEvents {
|
|
|
3094
3272
|
"replicator:mature": CustomEvent<ReplicatorMatureEvent>;
|
|
3095
3273
|
}
|
|
3096
3274
|
|
|
3275
|
+
export type SharedLogRuntimeSnapshot = Readonly<{
|
|
3276
|
+
nativeGraph: Readonly<{
|
|
3277
|
+
active: boolean;
|
|
3278
|
+
useHeads: boolean;
|
|
3279
|
+
}>;
|
|
3280
|
+
}>;
|
|
3281
|
+
|
|
3097
3282
|
@variant("shared_log")
|
|
3098
3283
|
export class SharedLog<
|
|
3099
3284
|
T,
|
|
@@ -3158,6 +3343,14 @@ export class SharedLog<
|
|
|
3158
3343
|
private _subscriptionChangeCallbacks?: Set<Promise<void>>;
|
|
3159
3344
|
private _acceptSubscriptionChangeCallbacks = false;
|
|
3160
3345
|
private _replicationLifecycleController?: AbortController;
|
|
3346
|
+
private _activeReceiveHandlersByPeer!: Map<string, PeerReceiveLeaseState>;
|
|
3347
|
+
private _receiveHandlerDrainByPeer!: Map<string, Set<Promise<void>>>;
|
|
3348
|
+
private _receiveCleanupGateByPeer!: Map<string, number>;
|
|
3349
|
+
private _subscriptionOpeningEpochByPeer!: Map<string, object>;
|
|
3350
|
+
private _openingSyncCapabilitiesByPeer!: Map<
|
|
3351
|
+
string,
|
|
3352
|
+
{ epoch: object; capabilities: number }
|
|
3353
|
+
>;
|
|
3161
3354
|
private _onFanoutDataFn?: (arg: any) => void;
|
|
3162
3355
|
private _onFanoutUnicastFn?: (arg: any) => void;
|
|
3163
3356
|
private _fanoutChannel?: FanoutChannel;
|
|
@@ -3173,6 +3366,7 @@ export class SharedLog<
|
|
|
3173
3366
|
private _closeController!: AbortController;
|
|
3174
3367
|
private _respondToIHaveTimeout!: any;
|
|
3175
3368
|
private _checkedPrune!: CheckedPruneCoordinator<T, R>;
|
|
3369
|
+
private _pendingIHaveCallbacks!: Set<Promise<void>>;
|
|
3176
3370
|
private _pendingIHaveExpiryTimer?: ReturnType<typeof setTimeout>;
|
|
3177
3371
|
private _pendingIHaveExpiryDeadline = Number.POSITIVE_INFINITY;
|
|
3178
3372
|
|
|
@@ -3204,6 +3398,18 @@ export class SharedLog<
|
|
|
3204
3398
|
{ attempts: number; timer?: ReturnType<typeof setTimeout> }
|
|
3205
3399
|
>;
|
|
3206
3400
|
private _replicationInfoApplyQueueByPeer!: Map<string, Promise<void>>;
|
|
3401
|
+
// Local receive generations fence replication-info handlers that were admitted
|
|
3402
|
+
// before a liveness eviction but reach the per-peer apply lane after it. Unlike
|
|
3403
|
+
// message timestamps, these tokens never compare clocks from different peers.
|
|
3404
|
+
private _replicationInfoReceiveEpochByPeer!: Map<string, object>;
|
|
3405
|
+
// Subscription callbacks can overlap because removing a replicator mutates the
|
|
3406
|
+
// replication index asynchronously. Keep that lifecycle separate from message
|
|
3407
|
+
// timestamps so a reconnect can synchronously revoke an older unsubscribe.
|
|
3408
|
+
private _subscriptionEpochByPeer!: Map<string, object>;
|
|
3409
|
+
// A superseded removal may be the queue item that actually observed an active
|
|
3410
|
+
// replicator. Carry that leave obligation to the transition that ultimately
|
|
3411
|
+
// wins, while a winning reconnect clears it without emitting a stale leave.
|
|
3412
|
+
private _pendingReplicatorLeaveByPeer!: Set<string>;
|
|
3207
3413
|
private _replicatorLivenessSweepRunning!: boolean;
|
|
3208
3414
|
private _replicatorLivenessTimer?: ReturnType<typeof setInterval>;
|
|
3209
3415
|
private _replicatorLivenessTargets!: string[];
|
|
@@ -4175,6 +4381,20 @@ export class SharedLog<
|
|
|
4175
4381
|
private _replicationAnnouncementRetryPending!: boolean;
|
|
4176
4382
|
private _replicationAnnouncementRetryGeneration!: number;
|
|
4177
4383
|
private _replicationAnnouncementRetryController!: AbortController;
|
|
4384
|
+
private replicationAnnouncementRepairDebounced:
|
|
4385
|
+
| ReturnType<typeof debounceFixedInterval>
|
|
4386
|
+
| undefined;
|
|
4387
|
+
private _replicationAnnouncementRepairPending!: boolean;
|
|
4388
|
+
private _replicationAnnouncementRepairGeneration!: number;
|
|
4389
|
+
private _replicationAnnouncementRepairGenerationController!: AbortController;
|
|
4390
|
+
private _replicationAnnouncementRepairTargets!: Map<
|
|
4391
|
+
string,
|
|
4392
|
+
ReplicationAnnouncementRepairTarget
|
|
4393
|
+
>;
|
|
4394
|
+
private _replicationAnnouncementRepairCohortSelected!: boolean;
|
|
4395
|
+
private _replicationAnnouncementRepairFairCursorHash!: string | undefined;
|
|
4396
|
+
private _replicationAnnouncementRepairMaxAttempts!: number;
|
|
4397
|
+
private _replicationAnnouncementRepairController!: AbortController;
|
|
4178
4398
|
|
|
4179
4399
|
// A fn for debouncing the calls for pruning
|
|
4180
4400
|
pruneDebouncedFn!: DebouncedAccumulatorMap<{
|
|
@@ -4210,6 +4430,7 @@ export class SharedLog<
|
|
|
4210
4430
|
private _repairSweepRunning!: boolean;
|
|
4211
4431
|
private _repairSweepPendingModes!: Set<RepairDispatchMode>;
|
|
4212
4432
|
private _repairSweepPendingPeersByMode!: Map<RepairDispatchMode, Set<string>>;
|
|
4433
|
+
private _repairSweepJoinWarmupGenerationByTarget!: Map<string, object>;
|
|
4213
4434
|
private _repairFrontierByMode!: Map<
|
|
4214
4435
|
RepairDispatchMode,
|
|
4215
4436
|
Map<string, Map<string, RepairDispatchEntry<R>>>
|
|
@@ -4222,10 +4443,24 @@ export class SharedLog<
|
|
|
4222
4443
|
RepairDispatchMode,
|
|
4223
4444
|
Set<string>
|
|
4224
4445
|
>;
|
|
4446
|
+
private _joinWarmupGenerationByTarget!: Map<string, object>;
|
|
4447
|
+
private _joinWarmupSendStateByTarget!: Map<
|
|
4448
|
+
string,
|
|
4449
|
+
JoinWarmupSendState<R>
|
|
4450
|
+
>;
|
|
4451
|
+
private _joinWarmupRetryTimersByTarget!: Map<
|
|
4452
|
+
string,
|
|
4453
|
+
Set<JoinWarmupRetryTimer>
|
|
4454
|
+
>;
|
|
4455
|
+
private _joinWarmupScheduledRetriesByTarget!: Map<
|
|
4456
|
+
string,
|
|
4457
|
+
JoinWarmupScheduledRetries<R>
|
|
4458
|
+
>;
|
|
4225
4459
|
private _repairSweepOptimisticGidPeersPending!: Map<
|
|
4226
4460
|
string,
|
|
4227
|
-
Map<string,
|
|
4461
|
+
Map<string, RepairSweepOptimisticPeerState>
|
|
4228
4462
|
>;
|
|
4463
|
+
private _repairSweepOptimisticGidsByPeer!: Map<string, Set<string>>;
|
|
4229
4464
|
private _entryKnownPeers!: Map<string, Set<string>>;
|
|
4230
4465
|
private _entryKnownPeerObservedAt!: Map<string, Map<string, number>>;
|
|
4231
4466
|
private _joinAuthoritativeRepairTimersByDelay!: Map<
|
|
@@ -4289,22 +4524,39 @@ export class SharedLog<
|
|
|
4289
4524
|
this.rpc = new RPC();
|
|
4290
4525
|
this._checkedPrune = new CheckedPruneCoordinator<T, R>();
|
|
4291
4526
|
this._pendingIHave = new Map();
|
|
4527
|
+
this._pendingIHaveCallbacks = new Set();
|
|
4292
4528
|
this.latestReplicationInfoMessage = new Map();
|
|
4293
4529
|
this._replicationInfoBlockedPeers = new Set();
|
|
4294
4530
|
this._replicationInfoRequestByPeer = new Map();
|
|
4295
4531
|
this._replicationInfoApplyQueueByPeer = new Map();
|
|
4532
|
+
this._replicationInfoReceiveEpochByPeer = new Map();
|
|
4533
|
+
this._subscriptionEpochByPeer = new Map();
|
|
4534
|
+
this._pendingReplicatorLeaveByPeer = new Set();
|
|
4535
|
+
this._activeReceiveHandlersByPeer = new Map();
|
|
4536
|
+
this._receiveHandlerDrainByPeer = new Map();
|
|
4537
|
+
this._receiveCleanupGateByPeer = new Map();
|
|
4538
|
+
this._subscriptionOpeningEpochByPeer = new Map();
|
|
4539
|
+
this._openingSyncCapabilitiesByPeer = new Map();
|
|
4296
4540
|
this._gidPeersHistory = new Map();
|
|
4297
4541
|
this._repairRetryTimers = new Set();
|
|
4298
4542
|
this._recentRepairDispatch = new Map();
|
|
4299
4543
|
this._repairSweepRunning = false;
|
|
4300
4544
|
this._repairSweepPendingModes = new Set();
|
|
4301
4545
|
this._repairSweepPendingPeersByMode = createRepairPendingPeersByMode();
|
|
4546
|
+
this._repairSweepJoinWarmupGenerationByTarget = new Map();
|
|
4302
4547
|
this._repairFrontierByMode = createRepairFrontierByMode() as Map<
|
|
4303
4548
|
RepairDispatchMode,
|
|
4304
4549
|
Map<string, Map<string, RepairDispatchEntry<R>>>
|
|
4305
4550
|
>;
|
|
4306
4551
|
this._repairFrontierActiveTargetsByMode = createRepairActiveTargetsByMode();
|
|
4552
|
+
this._repairFrontierBypassKnownPeersByMode =
|
|
4553
|
+
createRepairFrontierBypassKnownPeersByMode();
|
|
4554
|
+
this._joinWarmupGenerationByTarget = new Map();
|
|
4555
|
+
this._joinWarmupSendStateByTarget = new Map();
|
|
4556
|
+
this._joinWarmupRetryTimersByTarget = new Map();
|
|
4557
|
+
this._joinWarmupScheduledRetriesByTarget = new Map();
|
|
4307
4558
|
this._repairSweepOptimisticGidPeersPending = new Map();
|
|
4559
|
+
this._repairSweepOptimisticGidsByPeer = new Map();
|
|
4308
4560
|
this._entryKnownPeers = new Map();
|
|
4309
4561
|
this._joinAuthoritativeRepairTimersByDelay = new Map();
|
|
4310
4562
|
this._joinAuthoritativeRepairPeersByDelay = new Map();
|
|
@@ -4327,6 +4579,16 @@ export class SharedLog<
|
|
|
4327
4579
|
this._replicationAnnouncementRetryPending = false;
|
|
4328
4580
|
this._replicationAnnouncementRetryGeneration = 0;
|
|
4329
4581
|
this._replicationAnnouncementRetryController = new AbortController();
|
|
4582
|
+
this._replicationAnnouncementRepairPending = false;
|
|
4583
|
+
this._replicationAnnouncementRepairGeneration = 0;
|
|
4584
|
+
this._replicationAnnouncementRepairGenerationController =
|
|
4585
|
+
new AbortController();
|
|
4586
|
+
this._replicationAnnouncementRepairTargets = new Map();
|
|
4587
|
+
this._replicationAnnouncementRepairCohortSelected = false;
|
|
4588
|
+
this._replicationAnnouncementRepairFairCursorHash = undefined;
|
|
4589
|
+
this._replicationAnnouncementRepairMaxAttempts =
|
|
4590
|
+
REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS;
|
|
4591
|
+
this._replicationAnnouncementRepairController = new AbortController();
|
|
4330
4592
|
this.pendingMaturity = new Map();
|
|
4331
4593
|
this._closeController = new AbortController();
|
|
4332
4594
|
}
|
|
@@ -4689,9 +4951,11 @@ export class SharedLog<
|
|
|
4689
4951
|
peer: string;
|
|
4690
4952
|
message: ExchangeHeadsMessage<any>;
|
|
4691
4953
|
payload: Uint8Array;
|
|
4954
|
+
priority?: number;
|
|
4692
4955
|
fanoutUnicastOptions?: { timeoutMs?: number; signal?: AbortSignal };
|
|
4693
4956
|
}): Promise<void> {
|
|
4694
|
-
const { peer, message, payload, fanoutUnicastOptions } =
|
|
4957
|
+
const { peer, message, payload, priority, fanoutUnicastOptions } =
|
|
4958
|
+
properties;
|
|
4695
4959
|
const hints = await this._getSortedRouteHints(peer);
|
|
4696
4960
|
const hasDirectHint = hints.some(
|
|
4697
4961
|
(hint) => hint.kind === "directstream-ack",
|
|
@@ -4708,6 +4972,7 @@ export class SharedLog<
|
|
|
4708
4972
|
redundancy: 1,
|
|
4709
4973
|
to: [peer],
|
|
4710
4974
|
}),
|
|
4975
|
+
priority,
|
|
4711
4976
|
});
|
|
4712
4977
|
return;
|
|
4713
4978
|
} catch {
|
|
@@ -4746,6 +5011,7 @@ export class SharedLog<
|
|
|
4746
5011
|
redundancy: 1,
|
|
4747
5012
|
to: [peer],
|
|
4748
5013
|
}),
|
|
5014
|
+
priority,
|
|
4749
5015
|
});
|
|
4750
5016
|
}
|
|
4751
5017
|
|
|
@@ -5146,6 +5412,7 @@ export class SharedLog<
|
|
|
5146
5412
|
peer,
|
|
5147
5413
|
message,
|
|
5148
5414
|
payload,
|
|
5415
|
+
priority: delivery.priority,
|
|
5149
5416
|
fanoutUnicastOptions,
|
|
5150
5417
|
});
|
|
5151
5418
|
})(),
|
|
@@ -5160,6 +5427,7 @@ export class SharedLog<
|
|
|
5160
5427
|
redundancy: 1,
|
|
5161
5428
|
to: nativeDeliveryPlan.silentTo,
|
|
5162
5429
|
}),
|
|
5430
|
+
priority: delivery.priority,
|
|
5163
5431
|
})
|
|
5164
5432
|
.catch((error) => logger.error(error));
|
|
5165
5433
|
}
|
|
@@ -5300,6 +5568,7 @@ export class SharedLog<
|
|
|
5300
5568
|
peer,
|
|
5301
5569
|
message,
|
|
5302
5570
|
payload,
|
|
5571
|
+
priority: delivery.priority,
|
|
5303
5572
|
fanoutUnicastOptions,
|
|
5304
5573
|
});
|
|
5305
5574
|
})(),
|
|
@@ -5311,6 +5580,7 @@ export class SharedLog<
|
|
|
5311
5580
|
this.rpc
|
|
5312
5581
|
.send(message, {
|
|
5313
5582
|
mode: new SilentDelivery({ redundancy: 1, to: silentTo }),
|
|
5583
|
+
priority: delivery.priority,
|
|
5314
5584
|
})
|
|
5315
5585
|
.catch((error) => logger.error(error));
|
|
5316
5586
|
}
|
|
@@ -5654,6 +5924,184 @@ export class SharedLog<
|
|
|
5654
5924
|
}
|
|
5655
5925
|
}
|
|
5656
5926
|
|
|
5927
|
+
private isPeerReceiveAdmissionOpen(
|
|
5928
|
+
peerHash: string,
|
|
5929
|
+
replicationLifecycleController: AbortController | undefined,
|
|
5930
|
+
subscriptionEpoch: object | null,
|
|
5931
|
+
options?: {
|
|
5932
|
+
allowReplicationInfoBlocked?: boolean;
|
|
5933
|
+
allowCleanupGate?: boolean;
|
|
5934
|
+
},
|
|
5935
|
+
) {
|
|
5936
|
+
return (
|
|
5937
|
+
this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
5938
|
+
this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch) &&
|
|
5939
|
+
(options?.allowReplicationInfoBlocked === true ||
|
|
5940
|
+
!this._replicationInfoBlockedPeers.has(peerHash)) &&
|
|
5941
|
+
(options?.allowCleanupGate === true ||
|
|
5942
|
+
(this._receiveCleanupGateByPeer.get(peerHash) ?? 0) === 0)
|
|
5943
|
+
);
|
|
5944
|
+
}
|
|
5945
|
+
|
|
5946
|
+
private acquirePeerReceiveLease(
|
|
5947
|
+
peerHash: string,
|
|
5948
|
+
replicationLifecycleController: AbortController | undefined,
|
|
5949
|
+
subscriptionEpoch: object | null,
|
|
5950
|
+
options?: {
|
|
5951
|
+
allowReplicationInfoBlocked?: boolean;
|
|
5952
|
+
allowCleanupGate?: boolean;
|
|
5953
|
+
},
|
|
5954
|
+
): (() => void) | undefined {
|
|
5955
|
+
if (
|
|
5956
|
+
!this.isPeerReceiveAdmissionOpen(
|
|
5957
|
+
peerHash,
|
|
5958
|
+
replicationLifecycleController,
|
|
5959
|
+
subscriptionEpoch,
|
|
5960
|
+
options,
|
|
5961
|
+
)
|
|
5962
|
+
) {
|
|
5963
|
+
return;
|
|
5964
|
+
}
|
|
5965
|
+
|
|
5966
|
+
let state = this._activeReceiveHandlersByPeer.get(peerHash);
|
|
5967
|
+
if (!state) {
|
|
5968
|
+
const current: PeerReceiveLeaseBucket = { active: 0 };
|
|
5969
|
+
state = { current, activeBuckets: new Set() };
|
|
5970
|
+
this._activeReceiveHandlersByPeer.set(peerHash, state);
|
|
5971
|
+
}
|
|
5972
|
+
const bucket = state.current;
|
|
5973
|
+
bucket.active += 1;
|
|
5974
|
+
state.activeBuckets.add(bucket);
|
|
5975
|
+
let released = false;
|
|
5976
|
+
return () => {
|
|
5977
|
+
if (released) {
|
|
5978
|
+
return;
|
|
5979
|
+
}
|
|
5980
|
+
released = true;
|
|
5981
|
+
bucket.active -= 1;
|
|
5982
|
+
if (bucket.active > 0) {
|
|
5983
|
+
return;
|
|
5984
|
+
}
|
|
5985
|
+
state.activeBuckets.delete(bucket);
|
|
5986
|
+
bucket.drain?.resolve();
|
|
5987
|
+
if (
|
|
5988
|
+
state.activeBuckets.size === 0 &&
|
|
5989
|
+
state.current.active === 0 &&
|
|
5990
|
+
this._activeReceiveHandlersByPeer.get(peerHash) === state
|
|
5991
|
+
) {
|
|
5992
|
+
this._activeReceiveHandlersByPeer.delete(peerHash);
|
|
5993
|
+
}
|
|
5994
|
+
};
|
|
5995
|
+
}
|
|
5996
|
+
|
|
5997
|
+
private async drainPeerReceiveHandlers(peerHash: string): Promise<void> {
|
|
5998
|
+
const state = this._activeReceiveHandlersByPeer.get(peerHash);
|
|
5999
|
+
if (!state || state.activeBuckets.size === 0) {
|
|
6000
|
+
return;
|
|
6001
|
+
}
|
|
6002
|
+
|
|
6003
|
+
// Rotate before awaiting so a reconnect/opening generation can keep receiving
|
|
6004
|
+
// sync traffic without joining the drain for the previous subscription. Cleanup
|
|
6005
|
+
// callers gate admission first; terminal callers also repeat until empty.
|
|
6006
|
+
const buckets = [...state.activeBuckets];
|
|
6007
|
+
state.current = { active: 0 };
|
|
6008
|
+
const drain = Promise.all(
|
|
6009
|
+
buckets.map((bucket) => {
|
|
6010
|
+
bucket.drain ??= pDefer<void>();
|
|
6011
|
+
return bucket.drain.promise;
|
|
6012
|
+
}),
|
|
6013
|
+
).then(() => undefined);
|
|
6014
|
+
let drains = this._receiveHandlerDrainByPeer.get(peerHash);
|
|
6015
|
+
if (!drains) {
|
|
6016
|
+
drains = new Set();
|
|
6017
|
+
this._receiveHandlerDrainByPeer.set(peerHash, drains);
|
|
6018
|
+
}
|
|
6019
|
+
drains.add(drain);
|
|
6020
|
+
try {
|
|
6021
|
+
await drain;
|
|
6022
|
+
} finally {
|
|
6023
|
+
drains.delete(drain);
|
|
6024
|
+
if (drains.size === 0) {
|
|
6025
|
+
this._receiveHandlerDrainByPeer.delete(peerHash);
|
|
6026
|
+
}
|
|
6027
|
+
}
|
|
6028
|
+
}
|
|
6029
|
+
|
|
6030
|
+
private async drainReceiveHandlers(): Promise<void> {
|
|
6031
|
+
for (;;) {
|
|
6032
|
+
const peers = [...this._activeReceiveHandlersByPeer.keys()];
|
|
6033
|
+
if (peers.length === 0) {
|
|
6034
|
+
return;
|
|
6035
|
+
}
|
|
6036
|
+
await Promise.all(
|
|
6037
|
+
peers.map((peerHash) => this.drainPeerReceiveHandlers(peerHash)),
|
|
6038
|
+
);
|
|
6039
|
+
}
|
|
6040
|
+
}
|
|
6041
|
+
|
|
6042
|
+
private runPendingIHaveCallback(
|
|
6043
|
+
pending: PendingIHave<T>,
|
|
6044
|
+
entry: Entry<T>,
|
|
6045
|
+
): void {
|
|
6046
|
+
const replicationLifecycleController =
|
|
6047
|
+
this._replicationLifecycleController;
|
|
6048
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
6049
|
+
if (this._pendingIHave.get(entry.hash) === pending) {
|
|
6050
|
+
pending.clear();
|
|
6051
|
+
this._pendingIHave.delete(entry.hash);
|
|
6052
|
+
}
|
|
6053
|
+
return;
|
|
6054
|
+
}
|
|
6055
|
+
|
|
6056
|
+
// Register before invoking the callback so a synchronous terminal reentry
|
|
6057
|
+
// cannot make close/drop miss work that has already been admitted.
|
|
6058
|
+
const completion = pDefer<void>();
|
|
6059
|
+
const observed = completion.promise.catch((error) => {
|
|
6060
|
+
if (!(this.isTerminating() && isNotStartedError(error as Error))) {
|
|
6061
|
+
logger.error(error?.toString?.() ?? String(error));
|
|
6062
|
+
}
|
|
6063
|
+
});
|
|
6064
|
+
this._pendingIHaveCallbacks.add(observed);
|
|
6065
|
+
void observed.finally(() => {
|
|
6066
|
+
this._pendingIHaveCallbacks.delete(observed);
|
|
6067
|
+
if (this._pendingIHave.get(entry.hash) === pending) {
|
|
6068
|
+
pending.clear();
|
|
6069
|
+
this._pendingIHave.delete(entry.hash);
|
|
6070
|
+
}
|
|
6071
|
+
});
|
|
6072
|
+
|
|
6073
|
+
try {
|
|
6074
|
+
Promise.resolve(pending.callback(entry)).then(
|
|
6075
|
+
() => completion.resolve(),
|
|
6076
|
+
(error) => completion.reject(error),
|
|
6077
|
+
);
|
|
6078
|
+
} catch (error) {
|
|
6079
|
+
completion.reject(error);
|
|
6080
|
+
}
|
|
6081
|
+
}
|
|
6082
|
+
|
|
6083
|
+
private async drainPendingIHaveCallbacks(): Promise<void> {
|
|
6084
|
+
while (this._pendingIHaveCallbacks.size > 0) {
|
|
6085
|
+
await Promise.all([...this._pendingIHaveCallbacks]);
|
|
6086
|
+
}
|
|
6087
|
+
}
|
|
6088
|
+
|
|
6089
|
+
private blockPeerReceiveAdmission(peerHash: string) {
|
|
6090
|
+
this._receiveCleanupGateByPeer.set(
|
|
6091
|
+
peerHash,
|
|
6092
|
+
(this._receiveCleanupGateByPeer.get(peerHash) ?? 0) + 1,
|
|
6093
|
+
);
|
|
6094
|
+
}
|
|
6095
|
+
|
|
6096
|
+
private unblockPeerReceiveAdmission(peerHash: string) {
|
|
6097
|
+
const remaining = (this._receiveCleanupGateByPeer.get(peerHash) ?? 1) - 1;
|
|
6098
|
+
if (remaining > 0) {
|
|
6099
|
+
this._receiveCleanupGateByPeer.set(peerHash, remaining);
|
|
6100
|
+
} else {
|
|
6101
|
+
this._receiveCleanupGateByPeer.delete(peerHash);
|
|
6102
|
+
}
|
|
6103
|
+
}
|
|
6104
|
+
|
|
5657
6105
|
private handleReplicationLifecycleSendError(
|
|
5658
6106
|
error: unknown,
|
|
5659
6107
|
controller = this._replicationLifecycleController,
|
|
@@ -5765,11 +6213,301 @@ export class SharedLog<
|
|
|
5765
6213
|
);
|
|
5766
6214
|
}
|
|
5767
6215
|
|
|
6216
|
+
private setupReplicationAnnouncementRepairFunction(
|
|
6217
|
+
interval = REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL,
|
|
6218
|
+
maxAttempts = REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS,
|
|
6219
|
+
): void {
|
|
6220
|
+
if (!Number.isSafeInteger(maxAttempts) || maxAttempts <= 0) {
|
|
6221
|
+
throw new RangeError(
|
|
6222
|
+
"Replication announcement repair attempts must be positive",
|
|
6223
|
+
);
|
|
6224
|
+
}
|
|
6225
|
+
this.replicationAnnouncementRepairDebounced?.close();
|
|
6226
|
+
this._replicationAnnouncementRepairController?.abort();
|
|
6227
|
+
this._replicationAnnouncementRepairGenerationController?.abort();
|
|
6228
|
+
this._replicationAnnouncementRepairController = new AbortController();
|
|
6229
|
+
this._replicationAnnouncementRepairGenerationController =
|
|
6230
|
+
new AbortController();
|
|
6231
|
+
this._replicationAnnouncementRepairPending = false;
|
|
6232
|
+
this._replicationAnnouncementRepairGeneration =
|
|
6233
|
+
this._replicationAnnouncementRetryGeneration;
|
|
6234
|
+
this._replicationAnnouncementRepairTargets = new Map();
|
|
6235
|
+
this._replicationAnnouncementRepairCohortSelected = false;
|
|
6236
|
+
this._replicationAnnouncementRepairFairCursorHash = undefined;
|
|
6237
|
+
this._replicationAnnouncementRepairMaxAttempts = maxAttempts;
|
|
6238
|
+
this.replicationAnnouncementRepairDebounced = debounceFixedInterval(
|
|
6239
|
+
() => this.runCurrentReplicationStateAnnouncementRepair(),
|
|
6240
|
+
interval,
|
|
6241
|
+
{
|
|
6242
|
+
leading: false,
|
|
6243
|
+
// The wrapper catches worker failures while it still owns the generation
|
|
6244
|
+
// context. Keep this boundary visibility-only: it must never mutate a
|
|
6245
|
+
// possibly newer generation's pending state.
|
|
6246
|
+
onError: (error) => logger.error(error),
|
|
6247
|
+
},
|
|
6248
|
+
);
|
|
6249
|
+
}
|
|
6250
|
+
|
|
6251
|
+
private cancelCurrentReplicationStateAnnouncementRepair(): void {
|
|
6252
|
+
this._replicationAnnouncementRepairPending = false;
|
|
6253
|
+
this._replicationAnnouncementRepairController?.abort();
|
|
6254
|
+
this._replicationAnnouncementRepairGenerationController?.abort();
|
|
6255
|
+
this.replicationAnnouncementRepairDebounced?.close();
|
|
6256
|
+
this._replicationAnnouncementRepairTargets?.clear();
|
|
6257
|
+
}
|
|
6258
|
+
|
|
6259
|
+
private advanceCurrentReplicationStateAnnouncementRepairGeneration(): void {
|
|
6260
|
+
const generation = this._replicationAnnouncementRetryGeneration;
|
|
6261
|
+
if (generation === this._replicationAnnouncementRepairGeneration) {
|
|
6262
|
+
return;
|
|
6263
|
+
}
|
|
6264
|
+
|
|
6265
|
+
// Abort acknowledged sends carrying the old full-state snapshot before the
|
|
6266
|
+
// primary announcement for the new mutation waits on transport. Otherwise a
|
|
6267
|
+
// stale batch can hold the current state behind DirectStream's seek timeout.
|
|
6268
|
+
this._replicationAnnouncementRepairGenerationController?.abort();
|
|
6269
|
+
this._replicationAnnouncementRepairGenerationController =
|
|
6270
|
+
new AbortController();
|
|
6271
|
+
this._replicationAnnouncementRepairGeneration = generation;
|
|
6272
|
+
this._replicationAnnouncementRepairPending = false;
|
|
6273
|
+
this._replicationAnnouncementRepairTargets.clear();
|
|
6274
|
+
this._replicationAnnouncementRepairCohortSelected = false;
|
|
6275
|
+
}
|
|
6276
|
+
|
|
6277
|
+
private queueCurrentReplicationStateAnnouncementRepair(): void {
|
|
6278
|
+
if (
|
|
6279
|
+
this.closed ||
|
|
6280
|
+
this._closeController.signal.aborted ||
|
|
6281
|
+
this._replicationAnnouncementRepairController.signal.aborted ||
|
|
6282
|
+
!this.replicationAnnouncementRepairDebounced
|
|
6283
|
+
) {
|
|
6284
|
+
return;
|
|
6285
|
+
}
|
|
6286
|
+
|
|
6287
|
+
this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
|
|
6288
|
+
this._replicationAnnouncementRepairPending = true;
|
|
6289
|
+
void this.replicationAnnouncementRepairDebounced.call();
|
|
6290
|
+
}
|
|
6291
|
+
|
|
6292
|
+
private async runCurrentReplicationStateAnnouncementRepair(): Promise<void> {
|
|
6293
|
+
const generation = this._replicationAnnouncementRetryGeneration;
|
|
6294
|
+
const lifecycleController = this._replicationAnnouncementRepairController;
|
|
6295
|
+
const generationController =
|
|
6296
|
+
this._replicationAnnouncementRepairGenerationController;
|
|
6297
|
+
try {
|
|
6298
|
+
await this.repairCurrentReplicationStateAnnouncement({
|
|
6299
|
+
generation,
|
|
6300
|
+
lifecycleController,
|
|
6301
|
+
generationController,
|
|
6302
|
+
});
|
|
6303
|
+
} catch (error) {
|
|
6304
|
+
if (
|
|
6305
|
+
this.closed ||
|
|
6306
|
+
this._closeController.signal.aborted ||
|
|
6307
|
+
lifecycleController.signal.aborted ||
|
|
6308
|
+
generationController.signal.aborted ||
|
|
6309
|
+
generation !== this._replicationAnnouncementRetryGeneration ||
|
|
6310
|
+
generationController !==
|
|
6311
|
+
this._replicationAnnouncementRepairGenerationController
|
|
6312
|
+
) {
|
|
6313
|
+
return;
|
|
6314
|
+
}
|
|
6315
|
+
if (isNotStartedError(error as Error)) {
|
|
6316
|
+
return;
|
|
6317
|
+
}
|
|
6318
|
+
|
|
6319
|
+
// Only the worker that still owns the current generation may conclude
|
|
6320
|
+
// that its repair failed. A stale worker must not clear a newer call's
|
|
6321
|
+
// pending flag or attribute its error to the new generation.
|
|
6322
|
+
this._replicationAnnouncementRepairPending = false;
|
|
6323
|
+
logger.error(error);
|
|
6324
|
+
}
|
|
6325
|
+
}
|
|
6326
|
+
|
|
6327
|
+
private async repairCurrentReplicationStateAnnouncement(context?: {
|
|
6328
|
+
generation: number;
|
|
6329
|
+
lifecycleController: AbortController;
|
|
6330
|
+
generationController: AbortController;
|
|
6331
|
+
}): Promise<void> {
|
|
6332
|
+
if (!this._replicationAnnouncementRepairPending) {
|
|
6333
|
+
return;
|
|
6334
|
+
}
|
|
6335
|
+
const generation =
|
|
6336
|
+
context?.generation ?? this._replicationAnnouncementRetryGeneration;
|
|
6337
|
+
const lifecycleController =
|
|
6338
|
+
context?.lifecycleController ??
|
|
6339
|
+
this._replicationAnnouncementRepairController;
|
|
6340
|
+
const generationController =
|
|
6341
|
+
context?.generationController ??
|
|
6342
|
+
this._replicationAnnouncementRepairGenerationController;
|
|
6343
|
+
const segments = (await this.getMyReplicationSegments()).map((range) =>
|
|
6344
|
+
range.toReplicationRange(),
|
|
6345
|
+
);
|
|
6346
|
+
if (
|
|
6347
|
+
this.closed ||
|
|
6348
|
+
this._closeController.signal.aborted ||
|
|
6349
|
+
lifecycleController.signal.aborted ||
|
|
6350
|
+
generationController.signal.aborted
|
|
6351
|
+
) {
|
|
6352
|
+
return;
|
|
6353
|
+
}
|
|
6354
|
+
if (generation !== this._replicationAnnouncementRetryGeneration) {
|
|
6355
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
6356
|
+
return;
|
|
6357
|
+
}
|
|
6358
|
+
|
|
6359
|
+
const subscribers =
|
|
6360
|
+
(await this.node.services.pubsub.getSubscribers(this.topic)) ?? [];
|
|
6361
|
+
if (
|
|
6362
|
+
this.closed ||
|
|
6363
|
+
this._closeController.signal.aborted ||
|
|
6364
|
+
lifecycleController.signal.aborted ||
|
|
6365
|
+
generationController.signal.aborted
|
|
6366
|
+
) {
|
|
6367
|
+
return;
|
|
6368
|
+
}
|
|
6369
|
+
if (generation !== this._replicationAnnouncementRetryGeneration) {
|
|
6370
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
6371
|
+
return;
|
|
6372
|
+
}
|
|
6373
|
+
|
|
6374
|
+
const selfHash = this.node.identity.publicKey.hashcode();
|
|
6375
|
+
const currentTargets = new Map<string, PublicSignKey>();
|
|
6376
|
+
for (const key of subscribers) {
|
|
6377
|
+
const hash = key.hashcode();
|
|
6378
|
+
if (
|
|
6379
|
+
hash !== selfHash &&
|
|
6380
|
+
!this._replicationInfoBlockedPeers.has(hash) &&
|
|
6381
|
+
!currentTargets.has(hash)
|
|
6382
|
+
) {
|
|
6383
|
+
currentTargets.set(hash, key);
|
|
6384
|
+
}
|
|
6385
|
+
}
|
|
6386
|
+
|
|
6387
|
+
for (const [hash, target] of this._replicationAnnouncementRepairTargets) {
|
|
6388
|
+
if (target.generation !== generation || !currentTargets.has(hash)) {
|
|
6389
|
+
this._replicationAnnouncementRepairTargets.delete(hash);
|
|
6390
|
+
} else {
|
|
6391
|
+
target.key = currentTargets.get(hash)!;
|
|
6392
|
+
}
|
|
6393
|
+
}
|
|
6394
|
+
if (!this._replicationAnnouncementRepairCohortSelected) {
|
|
6395
|
+
const candidates = [...currentTargets.entries()].sort(([left], [right]) =>
|
|
6396
|
+
left.localeCompare(right),
|
|
6397
|
+
);
|
|
6398
|
+
const cursorIndex = this._replicationAnnouncementRepairFairCursorHash
|
|
6399
|
+
? candidates.findIndex(
|
|
6400
|
+
([hash]) =>
|
|
6401
|
+
hash.localeCompare(
|
|
6402
|
+
this._replicationAnnouncementRepairFairCursorHash!,
|
|
6403
|
+
) > 0,
|
|
6404
|
+
)
|
|
6405
|
+
: 0;
|
|
6406
|
+
const fairStart = cursorIndex < 0 ? 0 : cursorIndex;
|
|
6407
|
+
const fairOrder = [
|
|
6408
|
+
...candidates.slice(fairStart),
|
|
6409
|
+
...candidates.slice(0, fairStart),
|
|
6410
|
+
];
|
|
6411
|
+
const cohort = fairOrder.slice(
|
|
6412
|
+
0,
|
|
6413
|
+
REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION,
|
|
6414
|
+
);
|
|
6415
|
+
for (const [hash, key] of cohort) {
|
|
6416
|
+
this._replicationAnnouncementRepairTargets.set(hash, {
|
|
6417
|
+
key,
|
|
6418
|
+
generation,
|
|
6419
|
+
attempts: 0,
|
|
6420
|
+
done: false,
|
|
6421
|
+
});
|
|
6422
|
+
}
|
|
6423
|
+
if (cohort.length > 0) {
|
|
6424
|
+
this._replicationAnnouncementRepairFairCursorHash =
|
|
6425
|
+
cohort[cohort.length - 1][0];
|
|
6426
|
+
}
|
|
6427
|
+
this._replicationAnnouncementRepairCohortSelected = true;
|
|
6428
|
+
}
|
|
6429
|
+
|
|
6430
|
+
const batch = [
|
|
6431
|
+
...this._replicationAnnouncementRepairTargets.entries(),
|
|
6432
|
+
].filter(([, target]) => !target.done);
|
|
6433
|
+
const snapshot = new AllReplicatingSegmentsMessage({ segments });
|
|
6434
|
+
const results = await Promise.allSettled(
|
|
6435
|
+
batch.map(([, target]) =>
|
|
6436
|
+
this.rpc.send(snapshot, {
|
|
6437
|
+
mode: new AcknowledgeDelivery({
|
|
6438
|
+
to: [target.key],
|
|
6439
|
+
redundancy: 1,
|
|
6440
|
+
}),
|
|
6441
|
+
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
6442
|
+
signal: generationController.signal,
|
|
6443
|
+
}),
|
|
6444
|
+
),
|
|
6445
|
+
);
|
|
6446
|
+
if (
|
|
6447
|
+
this.closed ||
|
|
6448
|
+
this._closeController.signal.aborted ||
|
|
6449
|
+
lifecycleController.signal.aborted ||
|
|
6450
|
+
generationController.signal.aborted
|
|
6451
|
+
) {
|
|
6452
|
+
return;
|
|
6453
|
+
}
|
|
6454
|
+
if (generation !== this._replicationAnnouncementRetryGeneration) {
|
|
6455
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
6456
|
+
return;
|
|
6457
|
+
}
|
|
6458
|
+
|
|
6459
|
+
for (const [index, result] of results.entries()) {
|
|
6460
|
+
const [hash, attemptedTarget] = batch[index];
|
|
6461
|
+
const target = this._replicationAnnouncementRepairTargets.get(hash);
|
|
6462
|
+
if (target !== attemptedTarget || target.generation !== generation) {
|
|
6463
|
+
continue;
|
|
6464
|
+
}
|
|
6465
|
+
if (result.status === "fulfilled") {
|
|
6466
|
+
// DirectStream ACKs confirm that the signed transport envelope reached
|
|
6467
|
+
// the target. Applying the contained replication state remains a
|
|
6468
|
+
// receiver-local, best-effort operation.
|
|
6469
|
+
target.done = true;
|
|
6470
|
+
continue;
|
|
6471
|
+
}
|
|
6472
|
+
|
|
6473
|
+
target.attempts += 1;
|
|
6474
|
+
if (!isTransientReplicationAnnouncementRepairError(result.reason)) {
|
|
6475
|
+
target.done = true;
|
|
6476
|
+
logger.error(result.reason);
|
|
6477
|
+
} else if (
|
|
6478
|
+
target.attempts >= this._replicationAnnouncementRepairMaxAttempts
|
|
6479
|
+
) {
|
|
6480
|
+
target.done = true;
|
|
6481
|
+
logger.trace(
|
|
6482
|
+
"Acknowledged replication announcement repair exhausted for %s",
|
|
6483
|
+
hash,
|
|
6484
|
+
);
|
|
6485
|
+
}
|
|
6486
|
+
}
|
|
6487
|
+
|
|
6488
|
+
if (generation !== this._replicationAnnouncementRetryGeneration) {
|
|
6489
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
6490
|
+
return;
|
|
6491
|
+
}
|
|
6492
|
+
if (
|
|
6493
|
+
[...this._replicationAnnouncementRepairTargets.values()].some(
|
|
6494
|
+
(target) => !target.done,
|
|
6495
|
+
)
|
|
6496
|
+
) {
|
|
6497
|
+
void this.replicationAnnouncementRepairDebounced?.call();
|
|
6498
|
+
return;
|
|
6499
|
+
}
|
|
6500
|
+
|
|
6501
|
+
this._replicationAnnouncementRepairPending = false;
|
|
6502
|
+
this._replicationAnnouncementRepairTargets.clear();
|
|
6503
|
+
}
|
|
6504
|
+
|
|
5768
6505
|
private cancelCurrentReplicationStateAnnouncementRetry(): void {
|
|
5769
6506
|
this._replicationAnnouncementRetryGeneration += 1;
|
|
5770
6507
|
this._replicationAnnouncementRetryPending = false;
|
|
5771
6508
|
this._replicationAnnouncementRetryController?.abort();
|
|
5772
6509
|
this.replicationAnnouncementRetryDebounced?.close();
|
|
6510
|
+
this.cancelCurrentReplicationStateAnnouncementRepair();
|
|
5773
6511
|
}
|
|
5774
6512
|
|
|
5775
6513
|
private async sendReplicationAnnouncement(
|
|
@@ -5783,10 +6521,12 @@ export class SharedLog<
|
|
|
5783
6521
|
// local state; the generation mismatch forces one more current snapshot
|
|
5784
6522
|
// after that stale send settles.
|
|
5785
6523
|
this._replicationAnnouncementRetryGeneration += 1;
|
|
6524
|
+
this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
|
|
5786
6525
|
try {
|
|
5787
6526
|
await this.rpc.send(message, {
|
|
5788
6527
|
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
5789
6528
|
});
|
|
6529
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
5790
6530
|
} catch (error) {
|
|
5791
6531
|
// The local replication-index mutation precedes all calls to this
|
|
5792
6532
|
// wrapper. Preserve the explicit caller's rejection, but independently
|
|
@@ -5820,6 +6560,7 @@ export class SharedLog<
|
|
|
5820
6560
|
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
5821
6561
|
signal: controller.signal,
|
|
5822
6562
|
});
|
|
6563
|
+
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
5823
6564
|
} catch (error) {
|
|
5824
6565
|
if (
|
|
5825
6566
|
this.closed ||
|
|
@@ -6363,76 +7104,167 @@ export class SharedLog<
|
|
|
6363
7104
|
|
|
6364
7105
|
private async removeReplicator(
|
|
6365
7106
|
key: PublicSignKey | string,
|
|
6366
|
-
options?: {
|
|
6367
|
-
|
|
7107
|
+
options?: {
|
|
7108
|
+
cleanupIfSubscriptionSuperseded?: boolean;
|
|
7109
|
+
expectedJoinWarmupGeneration?: object | null;
|
|
7110
|
+
noEvent?: boolean;
|
|
7111
|
+
onRemoved?: (state: { wasReplicator: boolean }) => void;
|
|
7112
|
+
replicationLifecycleController?: AbortController;
|
|
7113
|
+
shouldRemove?: () => boolean;
|
|
7114
|
+
subscriptionEpoch?: object | null;
|
|
7115
|
+
},
|
|
7116
|
+
): Promise<boolean> {
|
|
6368
7117
|
const keyHash = typeof key === "string" ? key : key.hashcode();
|
|
6369
|
-
const
|
|
6370
|
-
|
|
6371
|
-
|
|
6372
|
-
|
|
6373
|
-
|
|
7118
|
+
const expectedJoinWarmupGeneration =
|
|
7119
|
+
options?.expectedJoinWarmupGeneration !== undefined
|
|
7120
|
+
? options.expectedJoinWarmupGeneration
|
|
7121
|
+
: (this._joinWarmupGenerationByTarget.get(keyHash) ?? null);
|
|
7122
|
+
const ownsSubscriptionEpoch = () =>
|
|
7123
|
+
options?.subscriptionEpoch === undefined ||
|
|
7124
|
+
this.isCurrentSubscriptionEpoch(keyHash, options.subscriptionEpoch);
|
|
7125
|
+
const ownsReplicationLifecycle = () =>
|
|
7126
|
+
options?.replicationLifecycleController === undefined ||
|
|
7127
|
+
this.isReplicationLifecycleActive(
|
|
7128
|
+
options.replicationLifecycleController,
|
|
7129
|
+
);
|
|
7130
|
+
const cancelExpectedJoinWarmupTarget = () => {
|
|
7131
|
+
if (
|
|
7132
|
+
expectedJoinWarmupGeneration !== null &&
|
|
7133
|
+
this._joinWarmupGenerationByTarget.get(keyHash) ===
|
|
7134
|
+
expectedJoinWarmupGeneration
|
|
7135
|
+
) {
|
|
7136
|
+
this.cancelJoinWarmupTarget(keyHash);
|
|
7137
|
+
}
|
|
7138
|
+
};
|
|
7139
|
+
const isMe = this.node.identity.publicKey.hashcode() === keyHash;
|
|
7140
|
+
let receiveAdmissionBlocked = false;
|
|
7141
|
+
const blockAndDrainPeerReceives = async () => {
|
|
7142
|
+
if (!receiveAdmissionBlocked) {
|
|
7143
|
+
this.blockPeerReceiveAdmission(keyHash);
|
|
7144
|
+
receiveAdmissionBlocked = true;
|
|
7145
|
+
}
|
|
7146
|
+
await this.drainPeerReceiveHandlers(keyHash);
|
|
7147
|
+
};
|
|
7148
|
+
const cleanupDisconnectedPeer = async () => {
|
|
7149
|
+
await blockAndDrainPeerReceives();
|
|
7150
|
+
this.removePeerFromGidPeerHistory(keyHash);
|
|
7151
|
+
this.cleanupPeerDisconnectTracking(keyHash);
|
|
7152
|
+
this.removeRepairFrontierTarget(keyHash, {
|
|
7153
|
+
expectedJoinWarmupGeneration,
|
|
7154
|
+
});
|
|
7155
|
+
this._recentRepairDispatch.delete(keyHash);
|
|
7156
|
+
if (!isMe) {
|
|
7157
|
+
await this.syncronizer.onPeerDisconnected(keyHash);
|
|
7158
|
+
}
|
|
7159
|
+
};
|
|
7160
|
+
let removed = false;
|
|
6374
7161
|
|
|
6375
|
-
|
|
6376
|
-
|
|
6377
|
-
|
|
6378
|
-
|
|
6379
|
-
|
|
6380
|
-
|
|
7162
|
+
// Replication-info updates already serialize per peer. Put the hash-wide
|
|
7163
|
+
// removal on the same queue so a newer reset cannot be deleted underneath
|
|
7164
|
+
// itself by an older unsubscribe callback.
|
|
7165
|
+
await this.withReplicationInfoApplyQueue(keyHash, async () => {
|
|
7166
|
+
if (!ownsReplicationLifecycle()) {
|
|
7167
|
+
return;
|
|
7168
|
+
}
|
|
7169
|
+
if (!ownsSubscriptionEpoch()) {
|
|
7170
|
+
// A reconnect may supersede an unsubscribe before its destructive
|
|
7171
|
+
// removal starts. Still retire the old connection's sync/request state
|
|
7172
|
+
// in lane order so the reconnect barrier cannot inherit stale caches.
|
|
7173
|
+
if (options?.cleanupIfSubscriptionSuperseded) {
|
|
7174
|
+
await cleanupDisconnectedPeer();
|
|
7175
|
+
}
|
|
7176
|
+
return;
|
|
7177
|
+
}
|
|
7178
|
+
if (options?.shouldRemove && !options.shouldRemove()) {
|
|
7179
|
+
return;
|
|
7180
|
+
}
|
|
7181
|
+
const wasReplicator = this.uniqueReplicators.has(keyHash);
|
|
6381
7182
|
|
|
6382
|
-
|
|
7183
|
+
const deleted = await this.replicationIndex
|
|
7184
|
+
.iterate({
|
|
7185
|
+
query: { hash: keyHash },
|
|
7186
|
+
})
|
|
7187
|
+
.all();
|
|
7188
|
+
// Liveness evidence can arrive without a subscription transition while
|
|
7189
|
+
// this scan is pending. Stop new receives, drain those already admitted,
|
|
7190
|
+
// then revalidate immediately before the first destructive mutation.
|
|
7191
|
+
await blockAndDrainPeerReceives();
|
|
7192
|
+
if (options?.shouldRemove && !options.shouldRemove()) {
|
|
7193
|
+
return;
|
|
7194
|
+
}
|
|
7195
|
+
// Liveness removal must not cancel a still-current warmup until fresh
|
|
7196
|
+
// activity has been rechecked at the destructive boundary.
|
|
7197
|
+
cancelExpectedJoinWarmupTarget();
|
|
6383
7198
|
|
|
6384
|
-
|
|
6385
|
-
|
|
6386
|
-
|
|
7199
|
+
await this.replicationIndex.del({ query: { hash: keyHash } });
|
|
7200
|
+
for (const result of deleted) {
|
|
7201
|
+
this.deleteNativeReplicationRange(result.value);
|
|
7202
|
+
}
|
|
6387
7203
|
|
|
6388
|
-
|
|
6389
|
-
|
|
6390
|
-
);
|
|
6391
|
-
|
|
7204
|
+
// Once the range indexes have removed the peer, make membership agree
|
|
7205
|
+
// before any later awaited bookkeeping can fail.
|
|
7206
|
+
this.uniqueReplicators.delete(keyHash);
|
|
7207
|
+
this._replicatorJoinEmitted.delete(keyHash);
|
|
6392
7208
|
|
|
6393
|
-
|
|
6394
|
-
|
|
6395
|
-
if (
|
|
6396
|
-
|
|
6397
|
-
|
|
6398
|
-
|
|
6399
|
-
}),
|
|
7209
|
+
await this.updateOldestTimestampFromIndex();
|
|
7210
|
+
|
|
7211
|
+
if (isMe) {
|
|
7212
|
+
// announce that we are no longer replicating
|
|
7213
|
+
|
|
7214
|
+
await this.sendReplicationAnnouncement(
|
|
7215
|
+
new AllReplicatingSegmentsMessage({ segments: [] }),
|
|
6400
7216
|
);
|
|
6401
|
-
} else {
|
|
6402
|
-
throw new Error("Key was not a PublicSignKey");
|
|
6403
7217
|
}
|
|
6404
|
-
}
|
|
6405
7218
|
|
|
6406
|
-
|
|
6407
|
-
|
|
6408
|
-
|
|
6409
|
-
|
|
6410
|
-
|
|
6411
|
-
|
|
6412
|
-
|
|
6413
|
-
|
|
7219
|
+
if (options?.noEvent !== true) {
|
|
7220
|
+
const publicKey = toLocalPublicSignKey(key);
|
|
7221
|
+
if (publicKey) {
|
|
7222
|
+
this.events.dispatchEvent(
|
|
7223
|
+
new CustomEvent<ReplicationChangeEvent>("replication:change", {
|
|
7224
|
+
detail: { publicKey },
|
|
7225
|
+
}),
|
|
7226
|
+
);
|
|
7227
|
+
} else {
|
|
7228
|
+
throw new Error("Key was not a PublicSignKey");
|
|
7229
|
+
}
|
|
7230
|
+
}
|
|
7231
|
+
const timestamp = BigInt(+new Date());
|
|
7232
|
+
for (const x of deleted) {
|
|
7233
|
+
this.replicationChangeDebounceFn.add({
|
|
7234
|
+
range: x.value,
|
|
7235
|
+
type: "removed",
|
|
7236
|
+
timestamp,
|
|
7237
|
+
});
|
|
7238
|
+
}
|
|
6414
7239
|
|
|
6415
|
-
|
|
6416
|
-
|
|
6417
|
-
|
|
6418
|
-
|
|
7240
|
+
const pendingMaturity = this.pendingMaturity.get(keyHash);
|
|
7241
|
+
if (pendingMaturity) {
|
|
7242
|
+
for (const [_k, v] of pendingMaturity) {
|
|
7243
|
+
clearTimeout(v.timeout);
|
|
7244
|
+
}
|
|
7245
|
+
this.pendingMaturity.delete(keyHash);
|
|
6419
7246
|
}
|
|
6420
|
-
this.pendingMaturity.delete(keyHash);
|
|
6421
|
-
}
|
|
6422
7247
|
|
|
6423
|
-
|
|
6424
|
-
|
|
6425
|
-
|
|
6426
|
-
this._checkedPrune.cleanupPeer(keyHash);
|
|
6427
|
-
this.removeRepairFrontierTarget(keyHash);
|
|
6428
|
-
this._recentRepairDispatch.delete(keyHash);
|
|
6429
|
-
if (!isMe) {
|
|
6430
|
-
this.syncronizer.onPeerDisconnected(keyHash);
|
|
6431
|
-
}
|
|
7248
|
+
// Keep local sync/prune state consistent even when a peer disappears
|
|
7249
|
+
// through replication-info updates without a topic unsubscribe event.
|
|
7250
|
+
await cleanupDisconnectedPeer();
|
|
6432
7251
|
|
|
6433
|
-
|
|
6434
|
-
|
|
6435
|
-
|
|
7252
|
+
if (!isMe) {
|
|
7253
|
+
// Replication-info handlers release their receive lease before joining
|
|
7254
|
+
// this lane. Fence every handler queued behind this successful removal,
|
|
7255
|
+
// regardless of whether it came from liveness, startup pruning, or an
|
|
7256
|
+
// unsubscribe transition.
|
|
7257
|
+
this.advanceReplicationInfoRecoveryEpoch(keyHash);
|
|
7258
|
+
this.rebalanceParticipationDebounced?.call();
|
|
7259
|
+
}
|
|
7260
|
+
removed = true;
|
|
7261
|
+
options?.onRemoved?.({ wasReplicator });
|
|
7262
|
+
}).finally(() => {
|
|
7263
|
+
if (receiveAdmissionBlocked) {
|
|
7264
|
+
this.unblockPeerReceiveAdmission(keyHash);
|
|
7265
|
+
}
|
|
7266
|
+
});
|
|
7267
|
+
return removed;
|
|
6436
7268
|
}
|
|
6437
7269
|
|
|
6438
7270
|
private async updateOldestTimestampFromIndex() {
|
|
@@ -6541,6 +7373,10 @@ export class SharedLog<
|
|
|
6541
7373
|
if (this._isTrustedReplicator && !(await this._isTrustedReplicator(from))) {
|
|
6542
7374
|
return undefined;
|
|
6543
7375
|
}
|
|
7376
|
+
// Preserve what the peer announced before duplicate filtering can empty the
|
|
7377
|
+
// working array. A repeated authoritative/non-empty announcement is still
|
|
7378
|
+
// proof of live membership after this process reopens.
|
|
7379
|
+
const announcedReplication = ranges.length > 0;
|
|
6544
7380
|
let isNewReplicator = false;
|
|
6545
7381
|
let timestamp = BigInt(ts ?? +new Date());
|
|
6546
7382
|
rebalance = rebalance == null ? true : rebalance;
|
|
@@ -6678,13 +7514,13 @@ export class SharedLog<
|
|
|
6678
7514
|
// means "nothing new", not "stopped". Removing the peer here would let
|
|
6679
7515
|
// uniqueReplicators diverge from replicationIndex while the peer's
|
|
6680
7516
|
// ranges stay indexed.
|
|
6681
|
-
const announcedStopped = reset === true &&
|
|
7517
|
+
const announcedStopped = reset === true && !announcedReplication;
|
|
6682
7518
|
const stoppedTransition = announcedStopped
|
|
6683
7519
|
? this.uniqueReplicators.delete(fromHash)
|
|
6684
7520
|
: false;
|
|
6685
7521
|
if (announcedStopped) {
|
|
6686
7522
|
this._replicatorJoinEmitted.delete(fromHash);
|
|
6687
|
-
} else if (
|
|
7523
|
+
} else if (announcedReplication) {
|
|
6688
7524
|
this.uniqueReplicators.add(fromHash);
|
|
6689
7525
|
}
|
|
6690
7526
|
|
|
@@ -6822,6 +7658,18 @@ export class SharedLog<
|
|
|
6822
7658
|
this.rebalanceParticipationDebounced?.call();
|
|
6823
7659
|
}
|
|
6824
7660
|
}
|
|
7661
|
+
if (
|
|
7662
|
+
announcedReplication &&
|
|
7663
|
+
!from.equals(this.node.identity.publicKey) &&
|
|
7664
|
+
!this._replicatorJoinEmitted.has(fromHash)
|
|
7665
|
+
) {
|
|
7666
|
+
this._replicatorJoinEmitted.add(fromHash);
|
|
7667
|
+
this.events.dispatchEvent(
|
|
7668
|
+
new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
|
|
7669
|
+
detail: { publicKey: from },
|
|
7670
|
+
}),
|
|
7671
|
+
);
|
|
7672
|
+
}
|
|
6825
7673
|
return diffs;
|
|
6826
7674
|
}
|
|
6827
7675
|
|
|
@@ -7068,21 +7916,50 @@ export class SharedLog<
|
|
|
7068
7916
|
return observedAt != null && Date.now() - observedAt <= maxAgeMs;
|
|
7069
7917
|
}
|
|
7070
7918
|
|
|
7071
|
-
private markRepairSweepOptimisticPeer(
|
|
7919
|
+
private markRepairSweepOptimisticPeer(
|
|
7920
|
+
gid: string,
|
|
7921
|
+
peer: string,
|
|
7922
|
+
generation: object,
|
|
7923
|
+
) {
|
|
7072
7924
|
let peers = this._repairSweepOptimisticGidPeersPending.get(gid);
|
|
7073
7925
|
if (!peers) {
|
|
7074
7926
|
peers = new Map();
|
|
7075
7927
|
this._repairSweepOptimisticGidPeersPending.set(gid, peers);
|
|
7076
7928
|
}
|
|
7077
|
-
|
|
7929
|
+
const current = peers.get(peer);
|
|
7930
|
+
peers.set(peer, {
|
|
7931
|
+
count: current?.generation === generation ? current.count + 1 : 1,
|
|
7932
|
+
generation,
|
|
7933
|
+
});
|
|
7934
|
+
let gids = this._repairSweepOptimisticGidsByPeer.get(peer);
|
|
7935
|
+
if (!gids) {
|
|
7936
|
+
gids = new Set();
|
|
7937
|
+
this._repairSweepOptimisticGidsByPeer.set(peer, gids);
|
|
7938
|
+
}
|
|
7939
|
+
gids.add(gid);
|
|
7078
7940
|
}
|
|
7079
7941
|
|
|
7080
7942
|
private hasPendingRepairSweepOptimisticPeer(gid: string, peer: string) {
|
|
7081
7943
|
return (
|
|
7082
|
-
(this._repairSweepOptimisticGidPeersPending.get(gid)?.get(peer) ||
|
|
7944
|
+
(this._repairSweepOptimisticGidPeersPending.get(gid)?.get(peer)?.count ||
|
|
7945
|
+
0) > 0
|
|
7083
7946
|
);
|
|
7084
7947
|
}
|
|
7085
7948
|
|
|
7949
|
+
private clearRepairSweepOptimisticPeer(peer: string) {
|
|
7950
|
+
for (const gid of this._repairSweepOptimisticGidsByPeer.get(peer) ?? []) {
|
|
7951
|
+
const peers = this._repairSweepOptimisticGidPeersPending.get(gid);
|
|
7952
|
+
if (!peers) {
|
|
7953
|
+
continue;
|
|
7954
|
+
}
|
|
7955
|
+
peers.delete(peer);
|
|
7956
|
+
if (peers.size === 0) {
|
|
7957
|
+
this._repairSweepOptimisticGidPeersPending.delete(gid);
|
|
7958
|
+
}
|
|
7959
|
+
}
|
|
7960
|
+
this._repairSweepOptimisticGidsByPeer.delete(peer);
|
|
7961
|
+
}
|
|
7962
|
+
|
|
7086
7963
|
private createEntryReplicatedForRepair(properties: {
|
|
7087
7964
|
entry: Entry<T>;
|
|
7088
7965
|
coordinates: NumberFromType<R>[];
|
|
@@ -7133,54 +8010,309 @@ export class SharedLog<
|
|
|
7133
8010
|
const timer = setTimeout(() => {
|
|
7134
8011
|
this._repairRetryTimers.delete(timer);
|
|
7135
8012
|
resolve();
|
|
7136
|
-
}, delayMs);
|
|
7137
|
-
timer.unref?.();
|
|
7138
|
-
this._repairRetryTimers.add(timer);
|
|
8013
|
+
}, delayMs);
|
|
8014
|
+
timer.unref?.();
|
|
8015
|
+
this._repairRetryTimers.add(timer);
|
|
8016
|
+
});
|
|
8017
|
+
}
|
|
8018
|
+
|
|
8019
|
+
private queueRepairFrontierEntries(
|
|
8020
|
+
mode: RepairDispatchMode,
|
|
8021
|
+
target: string,
|
|
8022
|
+
entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
|
|
8023
|
+
options?: { bypassKnownPeerHints?: boolean },
|
|
8024
|
+
) {
|
|
8025
|
+
let targets = this._repairFrontierByMode.get(mode);
|
|
8026
|
+
if (!targets) {
|
|
8027
|
+
targets = new Map();
|
|
8028
|
+
this._repairFrontierByMode.set(mode, targets);
|
|
8029
|
+
}
|
|
8030
|
+
let pending = targets.get(target);
|
|
8031
|
+
if (!pending) {
|
|
8032
|
+
pending = new Map();
|
|
8033
|
+
targets.set(target, pending);
|
|
8034
|
+
}
|
|
8035
|
+
for (const [hash, entry] of entries) {
|
|
8036
|
+
pending.set(hash, entry);
|
|
8037
|
+
}
|
|
8038
|
+
if (options?.bypassKnownPeerHints === true) {
|
|
8039
|
+
this._repairFrontierBypassKnownPeersByMode.get(mode)?.add(target);
|
|
8040
|
+
}
|
|
8041
|
+
}
|
|
8042
|
+
|
|
8043
|
+
private clearRepairFrontierHashes(target: string, hashes: Iterable<string>) {
|
|
8044
|
+
const hashList = [...hashes];
|
|
8045
|
+
if (hashList.length === 0) {
|
|
8046
|
+
return;
|
|
8047
|
+
}
|
|
8048
|
+
for (const mode of REPAIR_DISPATCH_MODES) {
|
|
8049
|
+
const pending = this._repairFrontierByMode.get(mode)?.get(target);
|
|
8050
|
+
if (!pending) {
|
|
8051
|
+
continue;
|
|
8052
|
+
}
|
|
8053
|
+
for (const hash of hashList) {
|
|
8054
|
+
pending.delete(hash);
|
|
8055
|
+
}
|
|
8056
|
+
if (pending.size === 0) {
|
|
8057
|
+
this._repairFrontierByMode.get(mode)?.delete(target);
|
|
8058
|
+
this._repairFrontierBypassKnownPeersByMode.get(mode)?.delete(target);
|
|
8059
|
+
}
|
|
8060
|
+
}
|
|
8061
|
+
}
|
|
8062
|
+
|
|
8063
|
+
private getJoinWarmupGeneration(target: string) {
|
|
8064
|
+
let generation = this._joinWarmupGenerationByTarget.get(target);
|
|
8065
|
+
if (!generation) {
|
|
8066
|
+
generation = {};
|
|
8067
|
+
this._joinWarmupGenerationByTarget.set(target, generation);
|
|
8068
|
+
}
|
|
8069
|
+
return generation;
|
|
8070
|
+
}
|
|
8071
|
+
|
|
8072
|
+
private trackJoinWarmupTimer(
|
|
8073
|
+
target: string,
|
|
8074
|
+
timer: JoinWarmupRetryTimer,
|
|
8075
|
+
) {
|
|
8076
|
+
let timers = this._joinWarmupRetryTimersByTarget.get(target);
|
|
8077
|
+
if (!timers) {
|
|
8078
|
+
timers = new Set();
|
|
8079
|
+
this._joinWarmupRetryTimersByTarget.set(target, timers);
|
|
8080
|
+
}
|
|
8081
|
+
timers.add(timer);
|
|
8082
|
+
this._repairRetryTimers.add(timer.handle);
|
|
8083
|
+
}
|
|
8084
|
+
|
|
8085
|
+
private untrackJoinWarmupTimer(
|
|
8086
|
+
target: string,
|
|
8087
|
+
timer: JoinWarmupRetryTimer,
|
|
8088
|
+
) {
|
|
8089
|
+
this._repairRetryTimers.delete(timer.handle);
|
|
8090
|
+
const timers = this._joinWarmupRetryTimersByTarget.get(target);
|
|
8091
|
+
if (!timers) {
|
|
8092
|
+
return;
|
|
8093
|
+
}
|
|
8094
|
+
timers.delete(timer);
|
|
8095
|
+
if (timers.size === 0) {
|
|
8096
|
+
this._joinWarmupRetryTimersByTarget.delete(target);
|
|
8097
|
+
}
|
|
8098
|
+
}
|
|
8099
|
+
|
|
8100
|
+
private cancelJoinWarmupTimers(target: string) {
|
|
8101
|
+
const timers = this._joinWarmupRetryTimersByTarget.get(target);
|
|
8102
|
+
if (!timers) {
|
|
8103
|
+
return;
|
|
8104
|
+
}
|
|
8105
|
+
for (const timer of [...timers]) {
|
|
8106
|
+
clearTimeout(timer.handle);
|
|
8107
|
+
timer.resolve?.();
|
|
8108
|
+
this.untrackJoinWarmupTimer(target, timer);
|
|
8109
|
+
}
|
|
8110
|
+
}
|
|
8111
|
+
|
|
8112
|
+
private cancelJoinWarmupTarget(target: string) {
|
|
8113
|
+
this._joinWarmupGenerationByTarget.delete(target);
|
|
8114
|
+
const pendingWarmupPeers =
|
|
8115
|
+
this._repairSweepPendingPeersByMode.get("join-warmup");
|
|
8116
|
+
pendingWarmupPeers?.delete(target);
|
|
8117
|
+
if (pendingWarmupPeers?.size === 0) {
|
|
8118
|
+
this._repairSweepPendingModes.delete("join-warmup");
|
|
8119
|
+
}
|
|
8120
|
+
this._repairSweepJoinWarmupGenerationByTarget.delete(target);
|
|
8121
|
+
this.clearRepairSweepOptimisticPeer(target);
|
|
8122
|
+
this.cancelJoinWarmupTimers(target);
|
|
8123
|
+
this._joinWarmupScheduledRetriesByTarget.delete(target);
|
|
8124
|
+
const state = this._joinWarmupSendStateByTarget.get(target);
|
|
8125
|
+
if (!state) {
|
|
8126
|
+
return;
|
|
8127
|
+
}
|
|
8128
|
+
state.bypassKnownPeerHints = false;
|
|
8129
|
+
state.entries.clear();
|
|
8130
|
+
state.pending = false;
|
|
8131
|
+
if (!state.running) {
|
|
8132
|
+
this._joinWarmupSendStateByTarget.delete(target);
|
|
8133
|
+
}
|
|
8134
|
+
}
|
|
8135
|
+
|
|
8136
|
+
private cancelAllJoinWarmupTargets() {
|
|
8137
|
+
const targets = new Set([
|
|
8138
|
+
...this._joinWarmupGenerationByTarget.keys(),
|
|
8139
|
+
...this._joinWarmupRetryTimersByTarget.keys(),
|
|
8140
|
+
...this._joinWarmupScheduledRetriesByTarget.keys(),
|
|
8141
|
+
...this._joinWarmupSendStateByTarget.keys(),
|
|
8142
|
+
]);
|
|
8143
|
+
for (const target of targets) {
|
|
8144
|
+
this.cancelJoinWarmupTarget(target);
|
|
8145
|
+
}
|
|
8146
|
+
}
|
|
8147
|
+
|
|
8148
|
+
private async sleepJoinWarmupTracked(target: string, delayMs: number) {
|
|
8149
|
+
if (delayMs <= 0) {
|
|
8150
|
+
return;
|
|
8151
|
+
}
|
|
8152
|
+
await new Promise<void>((resolve) => {
|
|
8153
|
+
let settled = false;
|
|
8154
|
+
let trackedTimer!: JoinWarmupRetryTimer;
|
|
8155
|
+
const settle = () => {
|
|
8156
|
+
if (settled) {
|
|
8157
|
+
return;
|
|
8158
|
+
}
|
|
8159
|
+
settled = true;
|
|
8160
|
+
this.untrackJoinWarmupTimer(target, trackedTimer);
|
|
8161
|
+
resolve();
|
|
8162
|
+
};
|
|
8163
|
+
const handle = setTimeout(settle, delayMs);
|
|
8164
|
+
handle.unref?.();
|
|
8165
|
+
trackedTimer = { handle, resolve: settle };
|
|
8166
|
+
this.trackJoinWarmupTimer(target, trackedTimer);
|
|
7139
8167
|
});
|
|
7140
8168
|
}
|
|
7141
8169
|
|
|
7142
|
-
private
|
|
7143
|
-
mode: RepairDispatchMode,
|
|
8170
|
+
private scheduleJoinWarmupRetries(
|
|
7144
8171
|
target: string,
|
|
8172
|
+
generation: object,
|
|
8173
|
+
delaysMs: Iterable<number>,
|
|
7145
8174
|
entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
|
|
7146
|
-
|
|
8175
|
+
bypassKnownPeerHints: boolean,
|
|
7147
8176
|
) {
|
|
7148
|
-
|
|
7149
|
-
|
|
7150
|
-
|
|
7151
|
-
|
|
8177
|
+
if (
|
|
8178
|
+
this.closed ||
|
|
8179
|
+
this._joinWarmupGenerationByTarget.get(target) !== generation
|
|
8180
|
+
) {
|
|
8181
|
+
return;
|
|
7152
8182
|
}
|
|
7153
|
-
let
|
|
7154
|
-
if (
|
|
7155
|
-
|
|
7156
|
-
|
|
8183
|
+
let scheduled = this._joinWarmupScheduledRetriesByTarget.get(target);
|
|
8184
|
+
if (scheduled?.generation !== generation) {
|
|
8185
|
+
this.cancelJoinWarmupTimers(target);
|
|
8186
|
+
this._joinWarmupScheduledRetriesByTarget.delete(target);
|
|
8187
|
+
scheduled = undefined;
|
|
7157
8188
|
}
|
|
7158
|
-
|
|
7159
|
-
|
|
8189
|
+
if (!scheduled) {
|
|
8190
|
+
scheduled = {
|
|
8191
|
+
generation,
|
|
8192
|
+
slotsByDelay: new Map(),
|
|
8193
|
+
};
|
|
8194
|
+
this._joinWarmupScheduledRetriesByTarget.set(target, scheduled);
|
|
7160
8195
|
}
|
|
7161
|
-
|
|
7162
|
-
|
|
8196
|
+
const delays = [...new Set(delaysMs)];
|
|
8197
|
+
const batch: JoinWarmupScheduledRetryBatch<R> = {
|
|
8198
|
+
bypassKnownPeerHints,
|
|
8199
|
+
entries: new Map(entries),
|
|
8200
|
+
remainingAttempts: delays.length,
|
|
8201
|
+
};
|
|
8202
|
+
const now = Date.now();
|
|
8203
|
+
for (const delayMs of delays) {
|
|
8204
|
+
let slot = scheduled.slotsByDelay.get(delayMs);
|
|
8205
|
+
if (!slot) {
|
|
8206
|
+
slot = { cohorts: [], head: 0 };
|
|
8207
|
+
scheduled.slotsByDelay.set(delayMs, slot);
|
|
8208
|
+
}
|
|
8209
|
+
const tail = slot.cohorts.at(-1);
|
|
8210
|
+
const dueAt = Math.max(tail?.dueAt ?? 0, now + delayMs);
|
|
8211
|
+
if (tail?.dueAt === dueAt) {
|
|
8212
|
+
tail.batches.push(batch);
|
|
8213
|
+
} else {
|
|
8214
|
+
slot.cohorts.push({
|
|
8215
|
+
batches: [batch],
|
|
8216
|
+
dueAt,
|
|
8217
|
+
});
|
|
8218
|
+
}
|
|
8219
|
+
this.armJoinWarmupRetrySlot(
|
|
8220
|
+
target,
|
|
8221
|
+
scheduled,
|
|
8222
|
+
delayMs,
|
|
8223
|
+
slot,
|
|
8224
|
+
);
|
|
7163
8225
|
}
|
|
7164
8226
|
}
|
|
7165
8227
|
|
|
7166
|
-
private
|
|
7167
|
-
|
|
7168
|
-
|
|
8228
|
+
private armJoinWarmupRetrySlot(
|
|
8229
|
+
target: string,
|
|
8230
|
+
scheduled: JoinWarmupScheduledRetries<R>,
|
|
8231
|
+
delayMs: number,
|
|
8232
|
+
slot: JoinWarmupScheduledRetrySlot<R>,
|
|
8233
|
+
) {
|
|
8234
|
+
const nextDueAt = slot.cohorts[slot.head]?.dueAt;
|
|
8235
|
+
if (nextDueAt == null) {
|
|
7169
8236
|
return;
|
|
7170
8237
|
}
|
|
7171
|
-
|
|
7172
|
-
|
|
7173
|
-
|
|
7174
|
-
|
|
7175
|
-
|
|
7176
|
-
|
|
7177
|
-
pending.delete(hash);
|
|
7178
|
-
}
|
|
7179
|
-
if (pending.size === 0) {
|
|
7180
|
-
this._repairFrontierByMode.get(mode)?.delete(target);
|
|
7181
|
-
this._repairFrontierBypassKnownPeersByMode.get(mode)?.delete(target);
|
|
7182
|
-
}
|
|
8238
|
+
if (slot.timer && slot.timerDueAt === nextDueAt) {
|
|
8239
|
+
return;
|
|
8240
|
+
}
|
|
8241
|
+
if (slot.timer) {
|
|
8242
|
+
clearTimeout(slot.timer.handle);
|
|
8243
|
+
this.untrackJoinWarmupTimer(target, slot.timer);
|
|
7183
8244
|
}
|
|
8245
|
+
let trackedTimer!: JoinWarmupRetryTimer;
|
|
8246
|
+
const handle = setTimeout(
|
|
8247
|
+
() => {
|
|
8248
|
+
this.untrackJoinWarmupTimer(target, trackedTimer);
|
|
8249
|
+
if (slot.timer !== trackedTimer) {
|
|
8250
|
+
return;
|
|
8251
|
+
}
|
|
8252
|
+
slot.timer = undefined;
|
|
8253
|
+
slot.timerDueAt = undefined;
|
|
8254
|
+
const current = this._joinWarmupScheduledRetriesByTarget.get(target);
|
|
8255
|
+
if (
|
|
8256
|
+
current !== scheduled ||
|
|
8257
|
+
current.slotsByDelay.get(delayMs) !== slot
|
|
8258
|
+
) {
|
|
8259
|
+
return;
|
|
8260
|
+
}
|
|
8261
|
+
|
|
8262
|
+
const dueEntries = new Map<string, RepairDispatchEntry<R>>();
|
|
8263
|
+
let bypassKnownPeerHints = false;
|
|
8264
|
+
const now = Date.now();
|
|
8265
|
+
while (
|
|
8266
|
+
slot.head < slot.cohorts.length &&
|
|
8267
|
+
slot.cohorts[slot.head].dueAt <= now
|
|
8268
|
+
) {
|
|
8269
|
+
const cohort = slot.cohorts[slot.head++];
|
|
8270
|
+
for (const batch of cohort.batches) {
|
|
8271
|
+
for (const [hash, entry] of batch.entries) {
|
|
8272
|
+
dueEntries.set(hash, entry);
|
|
8273
|
+
}
|
|
8274
|
+
bypassKnownPeerHints ||=
|
|
8275
|
+
batch.bypassKnownPeerHints;
|
|
8276
|
+
batch.remainingAttempts -= 1;
|
|
8277
|
+
if (batch.remainingAttempts === 0) {
|
|
8278
|
+
batch.entries.clear();
|
|
8279
|
+
}
|
|
8280
|
+
}
|
|
8281
|
+
cohort.batches.length = 0;
|
|
8282
|
+
}
|
|
8283
|
+
if (
|
|
8284
|
+
dueEntries.size > 0 &&
|
|
8285
|
+
!this.closed &&
|
|
8286
|
+
this._joinWarmupGenerationByTarget.get(target) ===
|
|
8287
|
+
scheduled.generation
|
|
8288
|
+
) {
|
|
8289
|
+
this.queueJoinWarmupSend(
|
|
8290
|
+
target,
|
|
8291
|
+
scheduled.generation,
|
|
8292
|
+
dueEntries,
|
|
8293
|
+
bypassKnownPeerHints,
|
|
8294
|
+
);
|
|
8295
|
+
}
|
|
8296
|
+
if (slot.head === slot.cohorts.length) {
|
|
8297
|
+
current.slotsByDelay.delete(delayMs);
|
|
8298
|
+
if (current.slotsByDelay.size === 0) {
|
|
8299
|
+
this._joinWarmupScheduledRetriesByTarget.delete(target);
|
|
8300
|
+
}
|
|
8301
|
+
return;
|
|
8302
|
+
}
|
|
8303
|
+
if (slot.head >= 1_024 && slot.head * 2 >= slot.cohorts.length) {
|
|
8304
|
+
slot.cohorts = slot.cohorts.slice(slot.head);
|
|
8305
|
+
slot.head = 0;
|
|
8306
|
+
}
|
|
8307
|
+
this.armJoinWarmupRetrySlot(target, current, delayMs, slot);
|
|
8308
|
+
},
|
|
8309
|
+
Math.max(0, nextDueAt - Date.now()),
|
|
8310
|
+
);
|
|
8311
|
+
handle.unref?.();
|
|
8312
|
+
trackedTimer = { handle };
|
|
8313
|
+
slot.timer = trackedTimer;
|
|
8314
|
+
slot.timerDueAt = nextDueAt;
|
|
8315
|
+
this.trackJoinWarmupTimer(target, trackedTimer);
|
|
7184
8316
|
}
|
|
7185
8317
|
|
|
7186
8318
|
private async getFullReplicaRepairCandidates(
|
|
@@ -7216,7 +8348,18 @@ export class SharedLog<
|
|
|
7216
8348
|
return candidates;
|
|
7217
8349
|
}
|
|
7218
8350
|
|
|
7219
|
-
private removeRepairFrontierTarget(
|
|
8351
|
+
private removeRepairFrontierTarget(
|
|
8352
|
+
target: string,
|
|
8353
|
+
options?: { expectedJoinWarmupGeneration?: object | null },
|
|
8354
|
+
) {
|
|
8355
|
+
if (
|
|
8356
|
+
options?.expectedJoinWarmupGeneration === undefined ||
|
|
8357
|
+
(options.expectedJoinWarmupGeneration !== null &&
|
|
8358
|
+
this._joinWarmupGenerationByTarget.get(target) ===
|
|
8359
|
+
options.expectedJoinWarmupGeneration)
|
|
8360
|
+
) {
|
|
8361
|
+
this.cancelJoinWarmupTarget(target);
|
|
8362
|
+
}
|
|
7220
8363
|
for (const mode of REPAIR_DISPATCH_MODES) {
|
|
7221
8364
|
this._repairFrontierByMode.get(mode)?.delete(target);
|
|
7222
8365
|
this._repairFrontierActiveTargetsByMode.get(mode)?.delete(target);
|
|
@@ -7329,6 +8472,110 @@ export class SharedLog<
|
|
|
7329
8472
|
});
|
|
7330
8473
|
}
|
|
7331
8474
|
|
|
8475
|
+
private queueJoinWarmupSend(
|
|
8476
|
+
target: string,
|
|
8477
|
+
generation: object,
|
|
8478
|
+
entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
|
|
8479
|
+
bypassKnownPeerHints: boolean,
|
|
8480
|
+
) {
|
|
8481
|
+
if (
|
|
8482
|
+
this.closed ||
|
|
8483
|
+
this._joinWarmupGenerationByTarget.get(target) !== generation
|
|
8484
|
+
) {
|
|
8485
|
+
return;
|
|
8486
|
+
}
|
|
8487
|
+
let state = this._joinWarmupSendStateByTarget.get(target);
|
|
8488
|
+
if (!state) {
|
|
8489
|
+
state = {
|
|
8490
|
+
bypassKnownPeerHints: false,
|
|
8491
|
+
entries: new Map(),
|
|
8492
|
+
generation,
|
|
8493
|
+
lastCompletedAt: Number.NEGATIVE_INFINITY,
|
|
8494
|
+
pending: false,
|
|
8495
|
+
running: false,
|
|
8496
|
+
};
|
|
8497
|
+
this._joinWarmupSendStateByTarget.set(target, state);
|
|
8498
|
+
} else if (state.generation !== generation) {
|
|
8499
|
+
state.bypassKnownPeerHints = false;
|
|
8500
|
+
state.entries.clear();
|
|
8501
|
+
state.pending = false;
|
|
8502
|
+
}
|
|
8503
|
+
for (const [hash, entry] of entries) {
|
|
8504
|
+
state.entries.set(hash, entry);
|
|
8505
|
+
}
|
|
8506
|
+
state.bypassKnownPeerHints ||= bypassKnownPeerHints;
|
|
8507
|
+
state.generation = generation;
|
|
8508
|
+
state.pending = true;
|
|
8509
|
+
if (state.running) {
|
|
8510
|
+
return;
|
|
8511
|
+
}
|
|
8512
|
+
void this.drainJoinWarmupSends(target, state).catch((error: any) =>
|
|
8513
|
+
logger.error(error),
|
|
8514
|
+
);
|
|
8515
|
+
}
|
|
8516
|
+
|
|
8517
|
+
private async drainJoinWarmupSends(
|
|
8518
|
+
target: string,
|
|
8519
|
+
state: JoinWarmupSendState<R>,
|
|
8520
|
+
) {
|
|
8521
|
+
if (state.running) {
|
|
8522
|
+
return;
|
|
8523
|
+
}
|
|
8524
|
+
state.running = true;
|
|
8525
|
+
try {
|
|
8526
|
+
while (state.pending) {
|
|
8527
|
+
state.pending = false;
|
|
8528
|
+
const generation = state.generation;
|
|
8529
|
+
const entries = new Map(state.entries);
|
|
8530
|
+
state.entries.clear();
|
|
8531
|
+
const bypassKnownPeerHints = state.bypassKnownPeerHints;
|
|
8532
|
+
state.bypassKnownPeerHints = false;
|
|
8533
|
+
const spacingMs = Math.max(
|
|
8534
|
+
0,
|
|
8535
|
+
state.lastCompletedAt + JOIN_WARMUP_SEND_SPACING_MS - Date.now(),
|
|
8536
|
+
);
|
|
8537
|
+
await this.sleepJoinWarmupTracked(target, spacingMs);
|
|
8538
|
+
if (
|
|
8539
|
+
this.closed ||
|
|
8540
|
+
state.generation !== generation ||
|
|
8541
|
+
this._joinWarmupGenerationByTarget.get(target) !== generation
|
|
8542
|
+
) {
|
|
8543
|
+
continue;
|
|
8544
|
+
}
|
|
8545
|
+
if (entries.size === 0) {
|
|
8546
|
+
continue;
|
|
8547
|
+
}
|
|
8548
|
+
this._repairMetrics["join-warmup"].simpleFallbackPasses += 1;
|
|
8549
|
+
try {
|
|
8550
|
+
await this.sendRepairEntriesWithTransport(
|
|
8551
|
+
target,
|
|
8552
|
+
entries,
|
|
8553
|
+
"simple",
|
|
8554
|
+
{
|
|
8555
|
+
bypassKnownPeers: bypassKnownPeerHints,
|
|
8556
|
+
bypassRecentKnownPeers: bypassKnownPeerHints,
|
|
8557
|
+
},
|
|
8558
|
+
);
|
|
8559
|
+
} catch (error: any) {
|
|
8560
|
+
logger.error(error);
|
|
8561
|
+
} finally {
|
|
8562
|
+
state.lastCompletedAt = Date.now();
|
|
8563
|
+
}
|
|
8564
|
+
}
|
|
8565
|
+
} finally {
|
|
8566
|
+
state.running = false;
|
|
8567
|
+
if (this._joinWarmupSendStateByTarget.get(target) === state) {
|
|
8568
|
+
if (state.pending) {
|
|
8569
|
+
void this.drainJoinWarmupSends(target, state).catch((error: any) =>
|
|
8570
|
+
logger.error(error),
|
|
8571
|
+
);
|
|
8572
|
+
} else if (!this._joinWarmupGenerationByTarget.has(target)) {
|
|
8573
|
+
this._joinWarmupSendStateByTarget.delete(target);
|
|
8574
|
+
}
|
|
8575
|
+
}
|
|
8576
|
+
}
|
|
8577
|
+
}
|
|
8578
|
+
|
|
7332
8579
|
private async sendMaybeMissingEntriesNow(
|
|
7333
8580
|
target: string,
|
|
7334
8581
|
entries: ReadonlyMap<string, RepairDispatchEntry<R>>,
|
|
@@ -7616,18 +8863,34 @@ export class SharedLog<
|
|
|
7616
8863
|
const bucket = this._repairMetrics[options.mode];
|
|
7617
8864
|
bucket.dispatches += 1;
|
|
7618
8865
|
bucket.entries += filteredEntries.size;
|
|
8866
|
+
const joinWarmupGeneration =
|
|
8867
|
+
options.mode === "join-warmup"
|
|
8868
|
+
? this.getJoinWarmupGeneration(target)
|
|
8869
|
+
: undefined;
|
|
8870
|
+
const bypassKnownPeerHints = this.shouldBypassKnownPeerHints(
|
|
8871
|
+
options.mode,
|
|
8872
|
+
options.bypassKnownPeerHints,
|
|
8873
|
+
);
|
|
7619
8874
|
|
|
7620
8875
|
const run = (transport: RepairTransportMode) => {
|
|
7621
|
-
if (
|
|
7622
|
-
|
|
7623
|
-
|
|
8876
|
+
if (
|
|
8877
|
+
transport === "simple" &&
|
|
8878
|
+
options.mode === "join-warmup" &&
|
|
8879
|
+
joinWarmupGeneration
|
|
8880
|
+
) {
|
|
8881
|
+
this.queueJoinWarmupSend(
|
|
8882
|
+
target,
|
|
8883
|
+
joinWarmupGeneration,
|
|
8884
|
+
filteredEntries,
|
|
8885
|
+
bypassKnownPeerHints,
|
|
8886
|
+
);
|
|
8887
|
+
return;
|
|
8888
|
+
}
|
|
8889
|
+
if (transport === "rateless") {
|
|
7624
8890
|
bucket.ratelessFirstPasses += 1;
|
|
8891
|
+
} else {
|
|
8892
|
+
bucket.simpleFallbackPasses += 1;
|
|
7625
8893
|
}
|
|
7626
|
-
const bypassKnownPeerHints = this.shouldBypassKnownPeerHints(
|
|
7627
|
-
options.mode,
|
|
7628
|
-
options.bypassKnownPeerHints,
|
|
7629
|
-
);
|
|
7630
|
-
|
|
7631
8894
|
return Promise.resolve(
|
|
7632
8895
|
this.sendRepairEntriesWithTransport(
|
|
7633
8896
|
target,
|
|
@@ -7641,12 +8904,21 @@ export class SharedLog<
|
|
|
7641
8904
|
).catch((error: any) => logger.error(error));
|
|
7642
8905
|
};
|
|
7643
8906
|
|
|
8907
|
+
const delayedJoinWarmupRetries: number[] = [];
|
|
7644
8908
|
retrySchedule.forEach((delayMs, index) => {
|
|
7645
8909
|
const transport = getRepairTransportForAttempt(options.mode, index);
|
|
7646
8910
|
if (delayMs === 0) {
|
|
7647
8911
|
void run(transport);
|
|
7648
8912
|
return;
|
|
7649
8913
|
}
|
|
8914
|
+
if (
|
|
8915
|
+
options.mode === "join-warmup" &&
|
|
8916
|
+
joinWarmupGeneration &&
|
|
8917
|
+
transport === "simple"
|
|
8918
|
+
) {
|
|
8919
|
+
delayedJoinWarmupRetries.push(delayMs);
|
|
8920
|
+
return;
|
|
8921
|
+
}
|
|
7650
8922
|
const timer = setTimeout(() => {
|
|
7651
8923
|
this._repairRetryTimers.delete(timer);
|
|
7652
8924
|
if (this.closed) {
|
|
@@ -7657,19 +8929,46 @@ export class SharedLog<
|
|
|
7657
8929
|
timer.unref?.();
|
|
7658
8930
|
this._repairRetryTimers.add(timer);
|
|
7659
8931
|
});
|
|
8932
|
+
if (joinWarmupGeneration && delayedJoinWarmupRetries.length > 0) {
|
|
8933
|
+
this.scheduleJoinWarmupRetries(
|
|
8934
|
+
target,
|
|
8935
|
+
joinWarmupGeneration,
|
|
8936
|
+
delayedJoinWarmupRetries,
|
|
8937
|
+
filteredEntries,
|
|
8938
|
+
bypassKnownPeerHints,
|
|
8939
|
+
);
|
|
8940
|
+
}
|
|
7660
8941
|
}
|
|
7661
8942
|
|
|
7662
8943
|
private scheduleRepairSweep(options: {
|
|
7663
8944
|
mode: RepairDispatchMode;
|
|
7664
8945
|
peers?: Iterable<string>;
|
|
8946
|
+
joinWarmupGenerations?: ReadonlyMap<string, object>;
|
|
7665
8947
|
}) {
|
|
7666
|
-
this._repairSweepPendingModes.add(options.mode);
|
|
7667
8948
|
const pendingPeers = this._repairSweepPendingPeersByMode.get(options.mode);
|
|
7668
8949
|
if (pendingPeers) {
|
|
7669
8950
|
for (const peer of options.peers ?? []) {
|
|
8951
|
+
if (options.mode === "join-warmup") {
|
|
8952
|
+
const generation =
|
|
8953
|
+
options.joinWarmupGenerations?.get(peer) ??
|
|
8954
|
+
this.getJoinWarmupGeneration(peer);
|
|
8955
|
+
if (
|
|
8956
|
+
this._joinWarmupGenerationByTarget.get(peer) !== generation
|
|
8957
|
+
) {
|
|
8958
|
+
continue;
|
|
8959
|
+
}
|
|
8960
|
+
this._repairSweepJoinWarmupGenerationByTarget.set(
|
|
8961
|
+
peer,
|
|
8962
|
+
generation,
|
|
8963
|
+
);
|
|
8964
|
+
}
|
|
7670
8965
|
pendingPeers.add(peer);
|
|
7671
8966
|
}
|
|
7672
8967
|
}
|
|
8968
|
+
if (!pendingPeers || pendingPeers.size === 0) {
|
|
8969
|
+
return;
|
|
8970
|
+
}
|
|
8971
|
+
this._repairSweepPendingModes.add(options.mode);
|
|
7673
8972
|
if (!this._repairSweepRunning && !this.closed) {
|
|
7674
8973
|
this._repairSweepRunning = true;
|
|
7675
8974
|
void this.runRepairSweep();
|
|
@@ -7731,10 +9030,31 @@ export class SharedLog<
|
|
|
7731
9030
|
const pendingPeersByMode = cloneRepairPendingPeersByMode(
|
|
7732
9031
|
this._repairSweepPendingPeersByMode,
|
|
7733
9032
|
);
|
|
9033
|
+
const pendingJoinWarmupGenerations = new Map(
|
|
9034
|
+
this._repairSweepJoinWarmupGenerationByTarget,
|
|
9035
|
+
);
|
|
7734
9036
|
this._repairSweepPendingModes.clear();
|
|
7735
9037
|
for (const peers of this._repairSweepPendingPeersByMode.values()) {
|
|
7736
9038
|
peers.clear();
|
|
7737
9039
|
}
|
|
9040
|
+
this._repairSweepJoinWarmupGenerationByTarget.clear();
|
|
9041
|
+
const pendingJoinWarmupPeers =
|
|
9042
|
+
pendingPeersByMode.get("join-warmup");
|
|
9043
|
+
const pruneStaleJoinWarmupPeers = () => {
|
|
9044
|
+
for (const peer of [...(pendingJoinWarmupPeers ?? [])]) {
|
|
9045
|
+
if (
|
|
9046
|
+
this._joinWarmupGenerationByTarget.get(peer) !==
|
|
9047
|
+
pendingJoinWarmupGenerations.get(peer)
|
|
9048
|
+
) {
|
|
9049
|
+
pendingJoinWarmupPeers?.delete(peer);
|
|
9050
|
+
}
|
|
9051
|
+
}
|
|
9052
|
+
if (pendingJoinWarmupPeers?.size === 0) {
|
|
9053
|
+
pendingModes.delete("join-warmup");
|
|
9054
|
+
}
|
|
9055
|
+
return pendingModes.size > 0;
|
|
9056
|
+
};
|
|
9057
|
+
pruneStaleJoinWarmupPeers();
|
|
7738
9058
|
|
|
7739
9059
|
if (pendingModes.size === 0) {
|
|
7740
9060
|
return;
|
|
@@ -7746,7 +9066,7 @@ export class SharedLog<
|
|
|
7746
9066
|
>();
|
|
7747
9067
|
const optimisticGidPeersConsumedByMode = new Map<
|
|
7748
9068
|
RepairDispatchMode,
|
|
7749
|
-
Map<string, Map<string,
|
|
9069
|
+
Map<string, Map<string, RepairSweepOptimisticPeerState>>
|
|
7750
9070
|
>();
|
|
7751
9071
|
for (const mode of pendingModes) {
|
|
7752
9072
|
const modePeers = pendingPeersByMode.get(mode);
|
|
@@ -7756,20 +9076,22 @@ export class SharedLog<
|
|
|
7756
9076
|
const optimisticGidPeers = new Map<string, Set<string>>();
|
|
7757
9077
|
const optimisticGidPeersConsumed = new Map<
|
|
7758
9078
|
string,
|
|
7759
|
-
Map<string,
|
|
9079
|
+
Map<string, RepairSweepOptimisticPeerState>
|
|
7760
9080
|
>();
|
|
7761
9081
|
for (const [gid, peerCounts] of this
|
|
7762
9082
|
._repairSweepOptimisticGidPeersPending) {
|
|
7763
9083
|
let matchedPeers: Set<string> | undefined;
|
|
7764
|
-
let matchedCounts:
|
|
7765
|
-
|
|
9084
|
+
let matchedCounts:
|
|
9085
|
+
| Map<string, RepairSweepOptimisticPeerState>
|
|
9086
|
+
| undefined;
|
|
9087
|
+
for (const [peer, state] of peerCounts) {
|
|
7766
9088
|
if (!modePeers.has(peer)) {
|
|
7767
9089
|
continue;
|
|
7768
9090
|
}
|
|
7769
9091
|
matchedPeers ||= new Set();
|
|
7770
9092
|
matchedCounts ||= new Map();
|
|
7771
9093
|
matchedPeers.add(peer);
|
|
7772
|
-
matchedCounts.set(peer,
|
|
9094
|
+
matchedCounts.set(peer, { ...state });
|
|
7773
9095
|
}
|
|
7774
9096
|
if (matchedPeers && matchedCounts) {
|
|
7775
9097
|
optimisticGidPeers.set(gid, matchedPeers);
|
|
@@ -7799,6 +9121,7 @@ export class SharedLog<
|
|
|
7799
9121
|
await this.getFullReplicaRepairCandidates(pendingRepairPeers, {
|
|
7800
9122
|
includeSubscribers: false,
|
|
7801
9123
|
});
|
|
9124
|
+
pruneStaleJoinWarmupPeers();
|
|
7802
9125
|
const fullReplicaRepairCandidateCount = Math.max(
|
|
7803
9126
|
1,
|
|
7804
9127
|
fullReplicaRepairCandidates.size,
|
|
@@ -7816,6 +9139,18 @@ export class SharedLog<
|
|
|
7816
9139
|
if (!entries || entries.size === 0) {
|
|
7817
9140
|
return;
|
|
7818
9141
|
}
|
|
9142
|
+
if (
|
|
9143
|
+
mode === "join-warmup" &&
|
|
9144
|
+
this._joinWarmupGenerationByTarget.get(target) !==
|
|
9145
|
+
pendingJoinWarmupGenerations.get(target)
|
|
9146
|
+
) {
|
|
9147
|
+
targets?.delete(target);
|
|
9148
|
+
pendingJoinWarmupPeers?.delete(target);
|
|
9149
|
+
if (pendingJoinWarmupPeers?.size === 0) {
|
|
9150
|
+
pendingModes.delete("join-warmup");
|
|
9151
|
+
}
|
|
9152
|
+
return;
|
|
9153
|
+
}
|
|
7819
9154
|
this.dispatchMaybeMissingEntries(target, entries, {
|
|
7820
9155
|
bypassRecentDedupe: true,
|
|
7821
9156
|
bypassKnownPeerHints:
|
|
@@ -7832,6 +9167,17 @@ export class SharedLog<
|
|
|
7832
9167
|
target: string,
|
|
7833
9168
|
entry: RepairDispatchEntry<any>,
|
|
7834
9169
|
) => {
|
|
9170
|
+
if (
|
|
9171
|
+
mode === "join-warmup" &&
|
|
9172
|
+
this._joinWarmupGenerationByTarget.get(target) !==
|
|
9173
|
+
pendingJoinWarmupGenerations.get(target)
|
|
9174
|
+
) {
|
|
9175
|
+
pendingJoinWarmupPeers?.delete(target);
|
|
9176
|
+
if (pendingJoinWarmupPeers?.size === 0) {
|
|
9177
|
+
pendingModes.delete("join-warmup");
|
|
9178
|
+
}
|
|
9179
|
+
return;
|
|
9180
|
+
}
|
|
7835
9181
|
const sweepTargets = nextFrontierByMode.get(mode);
|
|
7836
9182
|
if (sweepTargets) {
|
|
7837
9183
|
let sweepSet = sweepTargets.get(target);
|
|
@@ -7862,16 +9208,17 @@ export class SharedLog<
|
|
|
7862
9208
|
residentEntriesByHash &&
|
|
7863
9209
|
!this.hasCustomFindLeaders()
|
|
7864
9210
|
) {
|
|
7865
|
-
const repairDispatchPlan =
|
|
7866
|
-
{
|
|
9211
|
+
const repairDispatchPlan = pruneStaleJoinWarmupPeers()
|
|
9212
|
+
? await this.planResidentRepairDispatchBatch({
|
|
7867
9213
|
pendingModes,
|
|
7868
9214
|
pendingPeersByMode,
|
|
7869
9215
|
optimisticGidPeersByMode,
|
|
7870
9216
|
fullReplicaRepairCandidates,
|
|
7871
9217
|
fullReplicaRepairCandidateCount,
|
|
7872
9218
|
selfHash: this.node.identity.publicKey.hashcode(),
|
|
7873
|
-
|
|
7874
|
-
|
|
9219
|
+
})
|
|
9220
|
+
: new Map();
|
|
9221
|
+
pruneStaleJoinWarmupPeers();
|
|
7875
9222
|
for (const [mode, targets] of repairDispatchPlan) {
|
|
7876
9223
|
for (const [target, hashes] of targets) {
|
|
7877
9224
|
for (const hash of hashes) {
|
|
@@ -7882,13 +9229,20 @@ export class SharedLog<
|
|
|
7882
9229
|
}
|
|
7883
9230
|
}
|
|
7884
9231
|
}
|
|
7885
|
-
} else {
|
|
9232
|
+
} else if (pruneStaleJoinWarmupPeers()) {
|
|
7886
9233
|
const iterator = this.entryCoordinatesIndex.iterate({});
|
|
7887
9234
|
try {
|
|
7888
|
-
while (
|
|
9235
|
+
while (
|
|
9236
|
+
!this.closed &&
|
|
9237
|
+
!iterator.done() &&
|
|
9238
|
+
pruneStaleJoinWarmupPeers()
|
|
9239
|
+
) {
|
|
7889
9240
|
const entries = await iterator.next(
|
|
7890
9241
|
REPAIR_SWEEP_ENTRY_BATCH_SIZE,
|
|
7891
9242
|
);
|
|
9243
|
+
if (!pruneStaleJoinWarmupPeers()) {
|
|
9244
|
+
break;
|
|
9245
|
+
}
|
|
7892
9246
|
const entryReplicatedBatch = entries.map((entry) => entry.value);
|
|
7893
9247
|
const requestedReplicasBatch = entryReplicatedBatch.map((entry) =>
|
|
7894
9248
|
decodeReplicas(entry).getValue(this),
|
|
@@ -7903,6 +9257,9 @@ export class SharedLog<
|
|
|
7903
9257
|
fullReplicaRepairCandidateCount,
|
|
7904
9258
|
selfHash: this.node.identity.publicKey.hashcode(),
|
|
7905
9259
|
});
|
|
9260
|
+
if (!pruneStaleJoinWarmupPeers()) {
|
|
9261
|
+
break;
|
|
9262
|
+
}
|
|
7906
9263
|
const entriesByHash = new Map(
|
|
7907
9264
|
entryReplicatedBatch.map((entry) => [entry.hash, entry]),
|
|
7908
9265
|
);
|
|
@@ -7932,13 +9289,28 @@ export class SharedLog<
|
|
|
7932
9289
|
if (!pendingPeerCounts) {
|
|
7933
9290
|
continue;
|
|
7934
9291
|
}
|
|
7935
|
-
for (const [peer,
|
|
7936
|
-
const current = pendingPeerCounts.get(peer)
|
|
7937
|
-
|
|
9292
|
+
for (const [peer, consumed] of peerCounts) {
|
|
9293
|
+
const current = pendingPeerCounts.get(peer);
|
|
9294
|
+
if (
|
|
9295
|
+
!current ||
|
|
9296
|
+
current.generation !== consumed.generation
|
|
9297
|
+
) {
|
|
9298
|
+
continue;
|
|
9299
|
+
}
|
|
9300
|
+
const next = current.count - consumed.count;
|
|
7938
9301
|
if (next > 0) {
|
|
7939
|
-
pendingPeerCounts.set(peer,
|
|
9302
|
+
pendingPeerCounts.set(peer, {
|
|
9303
|
+
count: next,
|
|
9304
|
+
generation: current.generation,
|
|
9305
|
+
});
|
|
7940
9306
|
} else {
|
|
7941
9307
|
pendingPeerCounts.delete(peer);
|
|
9308
|
+
const gids =
|
|
9309
|
+
this._repairSweepOptimisticGidsByPeer.get(peer);
|
|
9310
|
+
gids?.delete(gid);
|
|
9311
|
+
if (gids?.size === 0) {
|
|
9312
|
+
this._repairSweepOptimisticGidsByPeer.delete(peer);
|
|
9313
|
+
}
|
|
7942
9314
|
}
|
|
7943
9315
|
}
|
|
7944
9316
|
if (pendingPeerCounts.size === 0) {
|
|
@@ -8513,10 +9885,22 @@ export class SharedLog<
|
|
|
8513
9885
|
return;
|
|
8514
9886
|
}
|
|
8515
9887
|
|
|
8516
|
-
|
|
9888
|
+
const pruneTasks = this.prune(toPrune);
|
|
9889
|
+
const confirmationTasks: Promise<void>[] = [];
|
|
8517
9890
|
for (const hash of responseStillApplies) {
|
|
8518
|
-
|
|
9891
|
+
const pendingDelete = this._checkedPrune.getPendingDelete(hash);
|
|
9892
|
+
if (pendingDelete) {
|
|
9893
|
+
confirmationTasks.push(
|
|
9894
|
+
Promise.resolve(pendingDelete.resolve(publicKeyHash)),
|
|
9895
|
+
);
|
|
9896
|
+
}
|
|
8519
9897
|
}
|
|
9898
|
+
// The restarted prune session owns its own bounded timeout and revalidates
|
|
9899
|
+
// before deletion. Only the peer-derived confirmations must remain inside
|
|
9900
|
+
// this receive lease; waiting for the whole prune session could stall close
|
|
9901
|
+
// or disconnect until its background timeout.
|
|
9902
|
+
void Promise.allSettled(pruneTasks);
|
|
9903
|
+
await Promise.allSettled(confirmationTasks);
|
|
8520
9904
|
}
|
|
8521
9905
|
|
|
8522
9906
|
async append(
|
|
@@ -12111,15 +13495,27 @@ export class SharedLog<
|
|
|
12111
13495
|
this._respondToIHaveTimeout = options?.respondToIHaveTimeout ?? 2e4;
|
|
12112
13496
|
this._checkedPrune = new CheckedPruneCoordinator<T, R>();
|
|
12113
13497
|
this._pendingIHave = new Map();
|
|
13498
|
+
this._pendingIHaveCallbacks = new Set();
|
|
12114
13499
|
this.latestReplicationInfoMessage = new Map();
|
|
12115
13500
|
this._replicationInfoBlockedPeers = new Set();
|
|
12116
13501
|
this._replicationInfoRequestByPeer = new Map();
|
|
13502
|
+
// Terminal close/drop drains the previous lifecycle before another open can
|
|
13503
|
+
// install fresh lanes and opaque per-subscription ownership tokens.
|
|
12117
13504
|
this._replicationInfoApplyQueueByPeer = new Map();
|
|
13505
|
+
this._replicationInfoReceiveEpochByPeer = new Map();
|
|
13506
|
+
this._subscriptionEpochByPeer = new Map();
|
|
13507
|
+
this._pendingReplicatorLeaveByPeer = new Set();
|
|
13508
|
+
this._activeReceiveHandlersByPeer = new Map();
|
|
13509
|
+
this._receiveHandlerDrainByPeer = new Map();
|
|
13510
|
+
this._receiveCleanupGateByPeer = new Map();
|
|
13511
|
+
this._subscriptionOpeningEpochByPeer = new Map();
|
|
13512
|
+
this._openingSyncCapabilitiesByPeer = new Map();
|
|
12118
13513
|
this._repairRetryTimers = new Set();
|
|
12119
13514
|
this._recentRepairDispatch = new Map();
|
|
12120
13515
|
this._repairSweepRunning = false;
|
|
12121
13516
|
this._repairSweepPendingModes = new Set();
|
|
12122
13517
|
this._repairSweepPendingPeersByMode = createRepairPendingPeersByMode();
|
|
13518
|
+
this._repairSweepJoinWarmupGenerationByTarget = new Map();
|
|
12123
13519
|
this._repairFrontierByMode = createRepairFrontierByMode() as Map<
|
|
12124
13520
|
RepairDispatchMode,
|
|
12125
13521
|
Map<string, Map<string, RepairDispatchEntry<R>>>
|
|
@@ -12127,7 +13523,12 @@ export class SharedLog<
|
|
|
12127
13523
|
this._repairFrontierActiveTargetsByMode = createRepairActiveTargetsByMode();
|
|
12128
13524
|
this._repairFrontierBypassKnownPeersByMode =
|
|
12129
13525
|
createRepairFrontierBypassKnownPeersByMode();
|
|
13526
|
+
this._joinWarmupGenerationByTarget = new Map();
|
|
13527
|
+
this._joinWarmupSendStateByTarget ??= new Map();
|
|
13528
|
+
this._joinWarmupRetryTimersByTarget = new Map();
|
|
13529
|
+
this._joinWarmupScheduledRetriesByTarget = new Map();
|
|
12130
13530
|
this._repairSweepOptimisticGidPeersPending = new Map();
|
|
13531
|
+
this._repairSweepOptimisticGidsByPeer = new Map();
|
|
12131
13532
|
this._entryKnownPeers = new Map();
|
|
12132
13533
|
this._entryKnownPeerObservedAt = new Map();
|
|
12133
13534
|
this._joinAuthoritativeRepairTimersByDelay = new Map();
|
|
@@ -12216,6 +13617,7 @@ export class SharedLog<
|
|
|
12216
13617
|
|
|
12217
13618
|
this._closeController = new AbortController();
|
|
12218
13619
|
this.setupReplicationAnnouncementRetryFunction();
|
|
13620
|
+
this.setupReplicationAnnouncementRepairFunction();
|
|
12219
13621
|
this._closeController.signal.addEventListener("abort", () => {
|
|
12220
13622
|
for (const [_peer, state] of this._replicationInfoRequestByPeer) {
|
|
12221
13623
|
if (state.timer) clearTimeout(state.timer);
|
|
@@ -12322,7 +13724,9 @@ export class SharedLog<
|
|
|
12322
13724
|
this.rpc.send(new BlocksMessage(message), options),
|
|
12323
13725
|
waitFor: this.rpc.waitFor.bind(this.rpc),
|
|
12324
13726
|
publicKey: this.node.identity.publicKey,
|
|
12325
|
-
|
|
13727
|
+
// Unsolicited block retention is opt-in. Explicit `true` retains the
|
|
13728
|
+
// compatible eager path with bounded validation and storage budgets.
|
|
13729
|
+
eagerBlocks: options?.eagerBlocks ?? false,
|
|
12326
13730
|
resolveProviders: async (cid, opts) => {
|
|
12327
13731
|
// 1) tracker-backed provider directory (best-effort, bounded)
|
|
12328
13732
|
try {
|
|
@@ -13280,11 +14684,17 @@ export class SharedLog<
|
|
|
13280
14684
|
async afterOpen(): Promise<void> {
|
|
13281
14685
|
await super.afterOpen();
|
|
13282
14686
|
const existingSubscribersPromise = this._getTopicSubscribers(this.topic);
|
|
14687
|
+
const replicationLifecycleController =
|
|
14688
|
+
this._replicationLifecycleController;
|
|
13283
14689
|
|
|
13284
14690
|
// We do this here, because these calls requires this.closed == false
|
|
13285
14691
|
void this.pruneOfflineReplicators()
|
|
13286
14692
|
.then(() => {
|
|
13287
|
-
|
|
14693
|
+
if (
|
|
14694
|
+
this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
14695
|
+
) {
|
|
14696
|
+
this._replicatorsReconciled = true;
|
|
14697
|
+
}
|
|
13288
14698
|
})
|
|
13289
14699
|
.catch((error) => {
|
|
13290
14700
|
if (isNotStartedError(error as Error)) {
|
|
@@ -13318,13 +14728,25 @@ export class SharedLog<
|
|
|
13318
14728
|
async pruneOfflineReplicators() {
|
|
13319
14729
|
// Go through all segments and wait for replicators to become reachable;
|
|
13320
14730
|
// otherwise prune them away from the local membership view.
|
|
14731
|
+
const replicationLifecycleController =
|
|
14732
|
+
this._replicationLifecycleController;
|
|
13321
14733
|
try {
|
|
14734
|
+
if (
|
|
14735
|
+
!replicationLifecycleController ||
|
|
14736
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
14737
|
+
) {
|
|
14738
|
+
return;
|
|
14739
|
+
}
|
|
13322
14740
|
const promises: Promise<any>[] = [];
|
|
13323
14741
|
const iterator = this.replicationIndex.iterate();
|
|
13324
14742
|
const checkedIsAlive = new Set<string>();
|
|
13325
14743
|
|
|
13326
14744
|
while (!iterator.done()) {
|
|
13327
|
-
|
|
14745
|
+
const segments = await iterator.next(1000);
|
|
14746
|
+
if (!this.isReplicationLifecycleActive(replicationLifecycleController)) {
|
|
14747
|
+
return;
|
|
14748
|
+
}
|
|
14749
|
+
for (const segment of segments) {
|
|
13328
14750
|
if (
|
|
13329
14751
|
checkedIsAlive.has(segment.value.hash) ||
|
|
13330
14752
|
this.node.identity.publicKey.hashcode() === segment.value.hash
|
|
@@ -13334,50 +14756,97 @@ export class SharedLog<
|
|
|
13334
14756
|
}
|
|
13335
14757
|
|
|
13336
14758
|
checkedIsAlive.add(segment.value.hash);
|
|
14759
|
+
const peerHash = segment.value.hash;
|
|
14760
|
+
const subscriptionEpoch = this.getSubscriptionEpoch(peerHash);
|
|
13337
14761
|
|
|
13338
14762
|
promises.push(
|
|
13339
|
-
waitForSubscribers(this.node,
|
|
14763
|
+
waitForSubscribers(this.node, peerHash, this.rpc.topic, {
|
|
13340
14764
|
timeout: this.waitForReplicatorTimeout,
|
|
13341
14765
|
signal: this._closeController.signal,
|
|
13342
14766
|
})
|
|
13343
14767
|
.then(async () => {
|
|
13344
|
-
|
|
13345
|
-
|
|
13346
|
-
|
|
14768
|
+
if (
|
|
14769
|
+
!this.isReplicationLifecycleActive(
|
|
14770
|
+
replicationLifecycleController,
|
|
14771
|
+
)
|
|
14772
|
+
) {
|
|
14773
|
+
return;
|
|
14774
|
+
}
|
|
14775
|
+
const key = await this._resolvePublicKeyFromHash(peerHash);
|
|
13347
14776
|
if (!key) {
|
|
13348
14777
|
throw new Error(
|
|
13349
14778
|
"Failed to resolve public key from hash: " +
|
|
13350
|
-
|
|
14779
|
+
peerHash,
|
|
13351
14780
|
);
|
|
13352
14781
|
}
|
|
13353
14782
|
|
|
13354
14783
|
const keyHash = key.hashcode();
|
|
13355
|
-
|
|
13356
|
-
|
|
13357
|
-
if (!this._replicatorJoinEmitted.has(keyHash)) {
|
|
13358
|
-
this._replicatorJoinEmitted.add(keyHash);
|
|
13359
|
-
this.events.dispatchEvent(
|
|
13360
|
-
new CustomEvent<ReplicatorJoinEvent>("replicator:join", {
|
|
13361
|
-
detail: { publicKey: key },
|
|
13362
|
-
}),
|
|
13363
|
-
);
|
|
13364
|
-
this.events.dispatchEvent(
|
|
13365
|
-
new CustomEvent<ReplicationChangeEvent>(
|
|
13366
|
-
"replication:change",
|
|
13367
|
-
{
|
|
13368
|
-
detail: { publicKey: key },
|
|
13369
|
-
},
|
|
13370
|
-
),
|
|
13371
|
-
);
|
|
14784
|
+
if (keyHash !== peerHash) {
|
|
14785
|
+
return;
|
|
13372
14786
|
}
|
|
14787
|
+
return this.withReplicationInfoApplyQueue(
|
|
14788
|
+
keyHash,
|
|
14789
|
+
async () => {
|
|
14790
|
+
// A successful reachability check may legitimately span a
|
|
14791
|
+
// subscribe event during startup. The current lane's blocked
|
|
14792
|
+
// state plus an extant index row are the authoritative guard;
|
|
14793
|
+
// only the destructive catch path remains tied to the old token.
|
|
14794
|
+
if (
|
|
14795
|
+
!this.isReplicationLifecycleActive(
|
|
14796
|
+
replicationLifecycleController,
|
|
14797
|
+
) ||
|
|
14798
|
+
this.closed ||
|
|
14799
|
+
this._replicationInfoBlockedPeers.has(keyHash)
|
|
14800
|
+
) {
|
|
14801
|
+
return;
|
|
14802
|
+
}
|
|
14803
|
+
const hasReplicationRange =
|
|
14804
|
+
(await this.replicationIndex.count({
|
|
14805
|
+
query: { hash: keyHash },
|
|
14806
|
+
})) > 0;
|
|
14807
|
+
if (
|
|
14808
|
+
!hasReplicationRange ||
|
|
14809
|
+
!this.isReplicationLifecycleActive(
|
|
14810
|
+
replicationLifecycleController,
|
|
14811
|
+
) ||
|
|
14812
|
+
this._replicationInfoBlockedPeers.has(keyHash)
|
|
14813
|
+
) {
|
|
14814
|
+
return;
|
|
14815
|
+
}
|
|
14816
|
+
this.uniqueReplicators.add(keyHash);
|
|
14817
|
+
|
|
14818
|
+
if (!this._replicatorJoinEmitted.has(keyHash)) {
|
|
14819
|
+
this._replicatorJoinEmitted.add(keyHash);
|
|
14820
|
+
this.events.dispatchEvent(
|
|
14821
|
+
new CustomEvent<ReplicatorJoinEvent>(
|
|
14822
|
+
"replicator:join",
|
|
14823
|
+
{ detail: { publicKey: key } },
|
|
14824
|
+
),
|
|
14825
|
+
);
|
|
14826
|
+
this.events.dispatchEvent(
|
|
14827
|
+
new CustomEvent<ReplicationChangeEvent>(
|
|
14828
|
+
"replication:change",
|
|
14829
|
+
{ detail: { publicKey: key } },
|
|
14830
|
+
),
|
|
14831
|
+
);
|
|
14832
|
+
}
|
|
14833
|
+
},
|
|
14834
|
+
);
|
|
13373
14835
|
})
|
|
13374
14836
|
.catch(async (error) => {
|
|
13375
|
-
if (
|
|
14837
|
+
if (
|
|
14838
|
+
isNotStartedError(error as Error) ||
|
|
14839
|
+
!this.isReplicationLifecycleActive(
|
|
14840
|
+
replicationLifecycleController,
|
|
14841
|
+
)
|
|
14842
|
+
) {
|
|
13376
14843
|
return;
|
|
13377
14844
|
}
|
|
13378
14845
|
|
|
13379
|
-
return this.removeReplicator(
|
|
14846
|
+
return this.removeReplicator(peerHash, {
|
|
13380
14847
|
noEvent: true,
|
|
14848
|
+
replicationLifecycleController,
|
|
14849
|
+
subscriptionEpoch,
|
|
13381
14850
|
});
|
|
13382
14851
|
}),
|
|
13383
14852
|
);
|
|
@@ -13386,7 +14855,12 @@ export class SharedLog<
|
|
|
13386
14855
|
|
|
13387
14856
|
return Promise.all(promises);
|
|
13388
14857
|
} catch (error) {
|
|
13389
|
-
if (
|
|
14858
|
+
if (
|
|
14859
|
+
isNotStartedError(error as Error) ||
|
|
14860
|
+
!this.isReplicationLifecycleActive(
|
|
14861
|
+
replicationLifecycleController,
|
|
14862
|
+
)
|
|
14863
|
+
) {
|
|
13390
14864
|
return;
|
|
13391
14865
|
}
|
|
13392
14866
|
throw error;
|
|
@@ -13458,11 +14932,28 @@ export class SharedLog<
|
|
|
13458
14932
|
this._replicatorLivenessFailures.delete(peerHash);
|
|
13459
14933
|
this._replicatorLastActivityAt.delete(peerHash);
|
|
13460
14934
|
this._peerSyncCapabilities.delete(peerHash);
|
|
14935
|
+
this.cleanupPendingIHavePeer(peerHash);
|
|
13461
14936
|
this._checkedPrune.cleanupPeer(peerHash);
|
|
13462
14937
|
}
|
|
13463
14938
|
|
|
14939
|
+
private cleanupPendingIHavePeer(peerHash: string) {
|
|
14940
|
+
for (const [hash, pending] of this._pendingIHave) {
|
|
14941
|
+
pending.requesting.delete(peerHash);
|
|
14942
|
+
if (pending.requesting.size === 0) {
|
|
14943
|
+
pending.clear();
|
|
14944
|
+
this._pendingIHave.delete(hash);
|
|
14945
|
+
}
|
|
14946
|
+
}
|
|
14947
|
+
}
|
|
14948
|
+
|
|
13464
14949
|
private markReplicatorActivity(peerHash: string, now = Date.now()) {
|
|
13465
14950
|
this._replicatorLastActivityAt.set(peerHash, now);
|
|
14951
|
+
// Any recent authenticated activity is positive liveness evidence. Reset the
|
|
14952
|
+
// consecutive miss streak immediately, including while an eviction is
|
|
14953
|
+
// waiting in the per-peer mutation lane.
|
|
14954
|
+
if (Date.now() - now < REPLICATOR_LIVENESS_IDLE_THRESHOLD_MS) {
|
|
14955
|
+
this._replicatorLivenessFailures.delete(peerHash);
|
|
14956
|
+
}
|
|
13466
14957
|
}
|
|
13467
14958
|
|
|
13468
14959
|
private hasRecentReplicatorActivity(peerHash: string, now = Date.now()) {
|
|
@@ -13477,39 +14968,65 @@ export class SharedLog<
|
|
|
13477
14968
|
return false;
|
|
13478
14969
|
}
|
|
13479
14970
|
|
|
14971
|
+
private advanceReplicationInfoRecoveryEpoch(peerHash: string) {
|
|
14972
|
+
// Handlers admitted before a successful peer removal must not restore state
|
|
14973
|
+
// when they eventually reach the apply lane. Reset the sender's
|
|
14974
|
+
// ordering watermark with the local epoch so a later arrival can be
|
|
14975
|
+
// accepted without comparing its clock to this receiver's clock.
|
|
14976
|
+
this.advanceReplicationInfoReceiveEpoch(peerHash);
|
|
14977
|
+
this.latestReplicationInfoMessage.delete(peerHash);
|
|
14978
|
+
}
|
|
14979
|
+
|
|
13480
14980
|
private async evictReplicatorFromLiveness(
|
|
13481
14981
|
peerHash: string,
|
|
13482
14982
|
publicKey: PublicSignKey,
|
|
14983
|
+
replicationLifecycleController: AbortController,
|
|
14984
|
+
subscriptionEpoch: object | null,
|
|
14985
|
+
observedActivityAt: number | undefined,
|
|
13483
14986
|
) {
|
|
13484
|
-
|
|
13485
|
-
|
|
13486
|
-
|
|
13487
|
-
|
|
13488
|
-
|
|
13489
|
-
|
|
14987
|
+
try {
|
|
14988
|
+
await this.removeReplicator(publicKey, {
|
|
14989
|
+
noEvent: true,
|
|
14990
|
+
replicationLifecycleController,
|
|
14991
|
+
shouldRemove: () =>
|
|
14992
|
+
this._replicatorLastActivityAt.get(peerHash) === observedActivityAt,
|
|
14993
|
+
subscriptionEpoch,
|
|
14994
|
+
onRemoved: ({ wasReplicator }) => {
|
|
14995
|
+
if (wasReplicator) {
|
|
14996
|
+
this._pendingReplicatorLeaveByPeer.add(peerHash);
|
|
14997
|
+
}
|
|
14998
|
+
// A newer subscription/lifecycle may have started while the admitted
|
|
14999
|
+
// removal was completing. Its reconnect barrier owns all later effects.
|
|
15000
|
+
if (
|
|
15001
|
+
!this.isReplicationLifecycleActive(
|
|
15002
|
+
replicationLifecycleController,
|
|
15003
|
+
) ||
|
|
15004
|
+
!this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch)
|
|
15005
|
+
) {
|
|
15006
|
+
return;
|
|
15007
|
+
}
|
|
15008
|
+
if (this._pendingReplicatorLeaveByPeer.delete(peerHash)) {
|
|
15009
|
+
this.events.dispatchEvent(
|
|
15010
|
+
new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
|
|
15011
|
+
detail: { publicKey },
|
|
15012
|
+
}),
|
|
15013
|
+
);
|
|
15014
|
+
}
|
|
13490
15015
|
|
|
13491
|
-
|
|
13492
|
-
|
|
15016
|
+
if (!this._replicationInfoBlockedPeers.has(peerHash)) {
|
|
15017
|
+
this.scheduleReplicationInfoRequests(
|
|
15018
|
+
publicKey,
|
|
15019
|
+
replicationLifecycleController,
|
|
15020
|
+
);
|
|
15021
|
+
}
|
|
15022
|
+
this._replicatorLivenessTargetsSize = -1;
|
|
15023
|
+
},
|
|
15024
|
+
});
|
|
13493
15025
|
} catch (error) {
|
|
13494
15026
|
if (!isNotStartedError(error as Error)) {
|
|
13495
15027
|
throw error;
|
|
13496
15028
|
}
|
|
13497
15029
|
}
|
|
13498
|
-
|
|
13499
|
-
this.cleanupPeerDisconnectTracking(peerHash);
|
|
13500
|
-
|
|
13501
|
-
if (wasReplicator) {
|
|
13502
|
-
this.events.dispatchEvent(
|
|
13503
|
-
new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
|
|
13504
|
-
detail: { publicKey },
|
|
13505
|
-
}),
|
|
13506
|
-
);
|
|
13507
|
-
}
|
|
13508
|
-
|
|
13509
|
-
if (!this._replicationInfoBlockedPeers.has(peerHash)) {
|
|
13510
|
-
this.scheduleReplicationInfoRequests(publicKey);
|
|
13511
|
-
}
|
|
13512
|
-
this._replicatorLivenessTargetsSize = -1;
|
|
13513
15030
|
}
|
|
13514
15031
|
|
|
13515
15032
|
private async resolveCandidatePeersForHash(
|
|
@@ -13589,7 +15106,13 @@ export class SharedLog<
|
|
|
13589
15106
|
}
|
|
13590
15107
|
|
|
13591
15108
|
private async runReplicatorLivenessSweep() {
|
|
13592
|
-
|
|
15109
|
+
const replicationLifecycleController =
|
|
15110
|
+
this._replicationLifecycleController;
|
|
15111
|
+
if (
|
|
15112
|
+
this.closed ||
|
|
15113
|
+
this._closeController.signal.aborted ||
|
|
15114
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
15115
|
+
) {
|
|
13593
15116
|
return;
|
|
13594
15117
|
}
|
|
13595
15118
|
if (this._replicatorLivenessSweepRunning) {
|
|
@@ -13615,14 +15138,30 @@ export class SharedLog<
|
|
|
13615
15138
|
logger.error((error as any)?.toString?.() ?? String(error));
|
|
13616
15139
|
}
|
|
13617
15140
|
} finally {
|
|
13618
|
-
|
|
15141
|
+
if (
|
|
15142
|
+
this._replicationLifecycleController ===
|
|
15143
|
+
replicationLifecycleController
|
|
15144
|
+
) {
|
|
15145
|
+
this._replicatorLivenessSweepRunning = false;
|
|
15146
|
+
}
|
|
13619
15147
|
}
|
|
13620
15148
|
}
|
|
13621
15149
|
|
|
13622
15150
|
private async probeReplicatorLiveness(peerHash: string) {
|
|
13623
|
-
|
|
15151
|
+
const replicationLifecycleController =
|
|
15152
|
+
this._replicationLifecycleController;
|
|
15153
|
+
if (
|
|
15154
|
+
this.closed ||
|
|
15155
|
+
this._closeController.signal.aborted ||
|
|
15156
|
+
!replicationLifecycleController ||
|
|
15157
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController)
|
|
15158
|
+
) {
|
|
13624
15159
|
return;
|
|
13625
15160
|
}
|
|
15161
|
+
const subscriptionEpoch = this.getSubscriptionEpoch(peerHash);
|
|
15162
|
+
const ownsProbe = () =>
|
|
15163
|
+
this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
15164
|
+
this.isCurrentSubscriptionEpoch(peerHash, subscriptionEpoch);
|
|
13626
15165
|
if (!this.uniqueReplicators.has(peerHash)) {
|
|
13627
15166
|
this._replicatorLivenessFailures.delete(peerHash);
|
|
13628
15167
|
return;
|
|
@@ -13630,18 +15169,36 @@ export class SharedLog<
|
|
|
13630
15169
|
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
13631
15170
|
return;
|
|
13632
15171
|
}
|
|
15172
|
+
const observedActivityAt = this._replicatorLastActivityAt.get(peerHash);
|
|
13633
15173
|
|
|
13634
15174
|
const publicKey = await this._resolvePublicKeyFromHash(peerHash);
|
|
15175
|
+
if (!ownsProbe()) {
|
|
15176
|
+
return;
|
|
15177
|
+
}
|
|
15178
|
+
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
15179
|
+
return;
|
|
15180
|
+
}
|
|
13635
15181
|
if (!publicKey) {
|
|
13636
15182
|
try {
|
|
13637
|
-
await this.removeReplicator(peerHash, {
|
|
15183
|
+
await this.removeReplicator(peerHash, {
|
|
15184
|
+
noEvent: true,
|
|
15185
|
+
replicationLifecycleController,
|
|
15186
|
+
shouldRemove: () =>
|
|
15187
|
+
this._replicatorLastActivityAt.get(peerHash) ===
|
|
15188
|
+
observedActivityAt,
|
|
15189
|
+
subscriptionEpoch,
|
|
15190
|
+
onRemoved: () => {
|
|
15191
|
+
if (!ownsProbe()) {
|
|
15192
|
+
return;
|
|
15193
|
+
}
|
|
15194
|
+
this._replicatorLivenessTargetsSize = -1;
|
|
15195
|
+
},
|
|
15196
|
+
});
|
|
13638
15197
|
} catch (error) {
|
|
13639
15198
|
if (!isNotStartedError(error as Error)) {
|
|
13640
15199
|
throw error;
|
|
13641
15200
|
}
|
|
13642
15201
|
}
|
|
13643
|
-
this.cleanupPeerDisconnectTracking(peerHash);
|
|
13644
|
-
this._replicatorLivenessTargetsSize = -1;
|
|
13645
15202
|
return;
|
|
13646
15203
|
}
|
|
13647
15204
|
|
|
@@ -13653,6 +15210,9 @@ export class SharedLog<
|
|
|
13653
15210
|
priority: ACK_CONTROL_PRIORITY,
|
|
13654
15211
|
responsePriority: ACK_CONTROL_PRIORITY,
|
|
13655
15212
|
});
|
|
15213
|
+
if (!ownsProbe()) {
|
|
15214
|
+
return;
|
|
15215
|
+
}
|
|
13656
15216
|
this.markReplicatorActivity(peerHash);
|
|
13657
15217
|
this._replicatorLivenessFailures.delete(peerHash);
|
|
13658
15218
|
return;
|
|
@@ -13661,29 +15221,53 @@ export class SharedLog<
|
|
|
13661
15221
|
return;
|
|
13662
15222
|
}
|
|
13663
15223
|
}
|
|
15224
|
+
if (!ownsProbe()) {
|
|
15225
|
+
return;
|
|
15226
|
+
}
|
|
15227
|
+
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
15228
|
+
return;
|
|
15229
|
+
}
|
|
13664
15230
|
|
|
13665
15231
|
// Relay-backed prod paths can keep a peer subscribed/reachable even if an
|
|
13666
15232
|
// ACKed liveness ping gets delayed or dropped under load. Treat observed
|
|
13667
15233
|
// topic presence as a positive liveness signal before evicting the peer.
|
|
13668
15234
|
if (await this.confirmReplicatorSubscriberPresence(peerHash)) {
|
|
15235
|
+
if (!ownsProbe()) {
|
|
15236
|
+
return;
|
|
15237
|
+
}
|
|
13669
15238
|
this.markReplicatorActivity(peerHash);
|
|
13670
15239
|
this._replicatorLivenessFailures.delete(peerHash);
|
|
13671
15240
|
return;
|
|
13672
15241
|
}
|
|
15242
|
+
if (!ownsProbe()) {
|
|
15243
|
+
return;
|
|
15244
|
+
}
|
|
15245
|
+
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
15246
|
+
return;
|
|
15247
|
+
}
|
|
13673
15248
|
|
|
13674
15249
|
const failures = (this._replicatorLivenessFailures.get(peerHash) ?? 0) + 1;
|
|
13675
15250
|
this._replicatorLivenessFailures.set(peerHash, failures);
|
|
13676
|
-
this.scheduleReplicationInfoRequests(
|
|
15251
|
+
this.scheduleReplicationInfoRequests(
|
|
15252
|
+
publicKey,
|
|
15253
|
+
replicationLifecycleController,
|
|
15254
|
+
);
|
|
13677
15255
|
|
|
13678
15256
|
if (failures < REPLICATOR_LIVENESS_PROBE_FAILURES_TO_EVICT) {
|
|
13679
15257
|
return;
|
|
13680
15258
|
}
|
|
13681
|
-
if (!this.uniqueReplicators.has(peerHash)) {
|
|
15259
|
+
if (!ownsProbe() || !this.uniqueReplicators.has(peerHash)) {
|
|
13682
15260
|
this._replicatorLivenessFailures.delete(peerHash);
|
|
13683
15261
|
return;
|
|
13684
15262
|
}
|
|
13685
15263
|
|
|
13686
|
-
await this.evictReplicatorFromLiveness(
|
|
15264
|
+
await this.evictReplicatorFromLiveness(
|
|
15265
|
+
peerHash,
|
|
15266
|
+
publicKey,
|
|
15267
|
+
replicationLifecycleController,
|
|
15268
|
+
subscriptionEpoch,
|
|
15269
|
+
observedActivityAt,
|
|
15270
|
+
);
|
|
13687
15271
|
}
|
|
13688
15272
|
|
|
13689
15273
|
private async confirmReplicatorSubscriberPresence(peerHash: string) {
|
|
@@ -13722,6 +15306,26 @@ export class SharedLog<
|
|
|
13722
15306
|
/* ((await this.log.entryIndex?.getMemoryUsage()) || 0) */ // + (await this.log.blocks.size())
|
|
13723
15307
|
}
|
|
13724
15308
|
|
|
15309
|
+
/** Return a detached snapshot of effective shared-log runtime settings. */
|
|
15310
|
+
getRuntimeSnapshot(): SharedLogRuntimeSnapshot {
|
|
15311
|
+
const nativeGraph = this.log.entryIndex.properties.nativeGraph;
|
|
15312
|
+
const active = nativeGraph?.graph != null;
|
|
15313
|
+
return Object.freeze({
|
|
15314
|
+
nativeGraph: Object.freeze({
|
|
15315
|
+
active,
|
|
15316
|
+
useHeads: active && nativeGraph?.useHeads === true,
|
|
15317
|
+
}),
|
|
15318
|
+
});
|
|
15319
|
+
}
|
|
15320
|
+
|
|
15321
|
+
/**
|
|
15322
|
+
* Return a detached snapshot of the optional eager-response cache.
|
|
15323
|
+
* Undefined means eager response retention is disabled for this log.
|
|
15324
|
+
*/
|
|
15325
|
+
getEagerBlockCacheTelemetry() {
|
|
15326
|
+
return this.remoteBlocks?.getEagerBlockCacheTelemetry();
|
|
15327
|
+
}
|
|
15328
|
+
|
|
13725
15329
|
private clampReplicas(value: number) {
|
|
13726
15330
|
const lower = this.replicas.min?.getValue(this) || 1;
|
|
13727
15331
|
const higher = this.replicas.max?.getValue(this) ?? Number.MAX_SAFE_INTEGER;
|
|
@@ -14477,6 +16081,7 @@ export class SharedLog<
|
|
|
14477
16081
|
),
|
|
14478
16082
|
);
|
|
14479
16083
|
captureSync(() => {
|
|
16084
|
+
this.cancelAllJoinWarmupTargets();
|
|
14480
16085
|
for (const timer of this._repairRetryTimers ?? []) clearTimeout(timer);
|
|
14481
16086
|
this._repairRetryTimers?.clear();
|
|
14482
16087
|
this._recentRepairDispatch?.clear();
|
|
@@ -14484,7 +16089,9 @@ export class SharedLog<
|
|
|
14484
16089
|
this._repairSweepPendingModes?.clear();
|
|
14485
16090
|
for (const peers of this._repairSweepPendingPeersByMode?.values() ?? [])
|
|
14486
16091
|
peers.clear();
|
|
16092
|
+
this._repairSweepJoinWarmupGenerationByTarget?.clear();
|
|
14487
16093
|
this._repairSweepOptimisticGidPeersPending?.clear();
|
|
16094
|
+
this._repairSweepOptimisticGidsByPeer?.clear();
|
|
14488
16095
|
this._entryKnownPeers?.clear();
|
|
14489
16096
|
this._entryKnownPeerObservedAt?.clear();
|
|
14490
16097
|
this._nativeSharedLogState?.clearEntryKnownPeers();
|
|
@@ -14521,7 +16128,14 @@ export class SharedLog<
|
|
|
14521
16128
|
}
|
|
14522
16129
|
captureSync(() => {
|
|
14523
16130
|
this._pendingIHave?.clear();
|
|
16131
|
+
this._pendingIHaveCallbacks?.clear();
|
|
14524
16132
|
this.latestReplicationInfoMessage?.clear();
|
|
16133
|
+
this._replicationInfoReceiveEpochByPeer?.clear();
|
|
16134
|
+
this._activeReceiveHandlersByPeer?.clear();
|
|
16135
|
+
this._receiveHandlerDrainByPeer?.clear();
|
|
16136
|
+
this._receiveCleanupGateByPeer?.clear();
|
|
16137
|
+
this._subscriptionOpeningEpochByPeer?.clear();
|
|
16138
|
+
this._openingSyncCapabilitiesByPeer?.clear();
|
|
14525
16139
|
this._gidPeersHistory?.clear();
|
|
14526
16140
|
this._peerSyncCapabilities?.clear();
|
|
14527
16141
|
this._liveRawGossipBatches?.clear();
|
|
@@ -14594,7 +16208,14 @@ export class SharedLog<
|
|
|
14594
16208
|
// SharedLog-specific await or observable teardown can admit a new owner.
|
|
14595
16209
|
this.preventParentAttachments();
|
|
14596
16210
|
this.stopSubscriptionChangeCallbackAdmission();
|
|
16211
|
+
this.cancelAllJoinWarmupTargets();
|
|
14597
16212
|
await this.drainSubscriptionChangeCallbacks();
|
|
16213
|
+
// An already-admitted subscription callback can create a fresh warmup
|
|
16214
|
+
// generation while the first cancellation is draining.
|
|
16215
|
+
this.cancelAllJoinWarmupTargets();
|
|
16216
|
+
await this.drainReceiveHandlers();
|
|
16217
|
+
await this.drainReplicationInfoApplyQueues();
|
|
16218
|
+
await this.drainPendingIHaveCallbacks();
|
|
14598
16219
|
this.ensureNativeDurabilityRuntimeState();
|
|
14599
16220
|
this.cancelCurrentReplicationStateAnnouncementRetry();
|
|
14600
16221
|
// Best-effort: announce that we are going offline before tearing down
|
|
@@ -14715,7 +16336,14 @@ export class SharedLog<
|
|
|
14715
16336
|
// the terminal fence only after that precondition succeeds.
|
|
14716
16337
|
this.preventParentAttachments();
|
|
14717
16338
|
this.stopSubscriptionChangeCallbackAdmission();
|
|
16339
|
+
this.cancelAllJoinWarmupTargets();
|
|
14718
16340
|
await this.drainSubscriptionChangeCallbacks();
|
|
16341
|
+
// An already-admitted subscription callback can create a fresh warmup
|
|
16342
|
+
// generation while the first cancellation is draining.
|
|
16343
|
+
this.cancelAllJoinWarmupTargets();
|
|
16344
|
+
await this.drainReceiveHandlers();
|
|
16345
|
+
await this.drainReplicationInfoApplyQueues();
|
|
16346
|
+
await this.drainPendingIHaveCallbacks();
|
|
14719
16347
|
this.cancelCurrentReplicationStateAnnouncementRetry();
|
|
14720
16348
|
// Best-effort: announce that we are going offline before tearing down
|
|
14721
16349
|
// RPC/subscription state (same reasoning as in `close()`).
|
|
@@ -14932,13 +16560,45 @@ export class SharedLog<
|
|
|
14932
16560
|
const stashBackedRawMessage = isStashBackedRawExchangeHeadsMessage(msg)
|
|
14933
16561
|
? msg
|
|
14934
16562
|
: undefined;
|
|
16563
|
+
let releasePeerReceiveLease: (() => void) | undefined;
|
|
14935
16564
|
try {
|
|
14936
16565
|
this.throwIfNativeDurableCommitFailed();
|
|
14937
16566
|
if (!context.from) {
|
|
14938
16567
|
throw new Error("Missing from in update role message");
|
|
14939
16568
|
}
|
|
16569
|
+
// Snapshot receive ownership before any async handler gets a chance to
|
|
16570
|
+
// yield. Replication-info messages reach their branch only after the
|
|
16571
|
+
// synchronizer declines them, and a U/S transition can happen meanwhile.
|
|
16572
|
+
const receiveFromHash = context.from.hashcode();
|
|
16573
|
+
const receiveReplicationLifecycleController =
|
|
16574
|
+
this._replicationLifecycleController;
|
|
16575
|
+
const receiveSubscriptionEpoch =
|
|
16576
|
+
this.getSubscriptionEpoch(receiveFromHash);
|
|
16577
|
+
const isOpeningSubscriptionReceive =
|
|
16578
|
+
this._subscriptionOpeningEpochByPeer.get(receiveFromHash) ===
|
|
16579
|
+
receiveSubscriptionEpoch;
|
|
16580
|
+
const isOpeningCapabilityAdvertisement =
|
|
16581
|
+
msg instanceof SyncCapabilitiesMessage && isOpeningSubscriptionReceive;
|
|
16582
|
+
releasePeerReceiveLease = this.acquirePeerReceiveLease(
|
|
16583
|
+
receiveFromHash,
|
|
16584
|
+
receiveReplicationLifecycleController,
|
|
16585
|
+
receiveSubscriptionEpoch,
|
|
16586
|
+
{
|
|
16587
|
+
// The replication-info fence existed before receive leases and is
|
|
16588
|
+
// intentionally narrower than the subscription itself. Keep admitting
|
|
16589
|
+
// sync negotiation/data while the new subscription drains the previous
|
|
16590
|
+
// apply generation; replication-info branches recheck the fence below.
|
|
16591
|
+
allowReplicationInfoBlocked: isOpeningSubscriptionReceive,
|
|
16592
|
+
allowCleanupGate: isOpeningCapabilityAdvertisement,
|
|
16593
|
+
},
|
|
16594
|
+
);
|
|
16595
|
+
if (!releasePeerReceiveLease) {
|
|
16596
|
+
return;
|
|
16597
|
+
}
|
|
16598
|
+
const receiveReplicationInfoReceiveEpoch =
|
|
16599
|
+
this.getReplicationInfoReceiveEpoch(receiveFromHash);
|
|
14940
16600
|
if (!context.from.equals(this.node.identity.publicKey)) {
|
|
14941
|
-
this.markReplicatorActivity(
|
|
16601
|
+
this.markReplicatorActivity(receiveFromHash);
|
|
14942
16602
|
}
|
|
14943
16603
|
|
|
14944
16604
|
if (msg instanceof ResponseRoleMessage) {
|
|
@@ -16269,6 +17929,7 @@ export class SharedLog<
|
|
|
16269
17929
|
);
|
|
16270
17930
|
return allToMergeMaterializedEntries;
|
|
16271
17931
|
};
|
|
17932
|
+
let admittedMergeHashes: ReadonlySet<string> = new Set();
|
|
16272
17933
|
let nativePreparedCommittedHashes: Set<string> | undefined;
|
|
16273
17934
|
if (allToMerge.length > 0) {
|
|
16274
17935
|
const validateStartedAt = syncProfileStart(syncProfile);
|
|
@@ -16377,9 +18038,9 @@ export class SharedLog<
|
|
|
16377
18038
|
const entriesToPersist = usedNativeAllKeptJoinPlan
|
|
16378
18039
|
? allToMergeShallowEntries
|
|
16379
18040
|
: joinPlans.flatMap((plan) => plan.toPersist);
|
|
16380
|
-
|
|
18041
|
+
let coordinatePersistFallbackEntries: ShallowOrFullEntry<any>[] =
|
|
16381
18042
|
[];
|
|
16382
|
-
|
|
18043
|
+
let reusableCoordinatePersistItems: CoordinatePersistBatchItem<R>[] =
|
|
16383
18044
|
[];
|
|
16384
18045
|
for (const entry of entriesToPersist) {
|
|
16385
18046
|
const reusablePlan = reusableCoordinatePlans.get(entry.hash);
|
|
@@ -16397,8 +18058,6 @@ export class SharedLog<
|
|
|
16397
18058
|
prepared: reusablePlan.prepared,
|
|
16398
18059
|
});
|
|
16399
18060
|
}
|
|
16400
|
-
const reusableCoordinatePersistItemCount =
|
|
16401
|
-
reusableCoordinatePersistItems.length;
|
|
16402
18061
|
let nativePreparedCoordinateBatch:
|
|
16403
18062
|
| NativeBackboneReceiveCoordinateBatch<R>
|
|
16404
18063
|
| undefined;
|
|
@@ -16501,6 +18160,34 @@ export class SharedLog<
|
|
|
16501
18160
|
__peerbitProfile: syncProfile,
|
|
16502
18161
|
});
|
|
16503
18162
|
}
|
|
18163
|
+
// A recursive lower-log join can resolve successfully while declining
|
|
18164
|
+
// an individual top-level entry (for example, when one of its parents
|
|
18165
|
+
// is temporarily unavailable). The public Log.join() API intentionally
|
|
18166
|
+
// does not expose that per-entry result, so make local index presence the
|
|
18167
|
+
// authority before publishing any SharedLog-side effects. A successful
|
|
18168
|
+
// prepared-facts batch is atomic and already proves every input hash.
|
|
18169
|
+
const admittedHashes = joinedPreparedFacts
|
|
18170
|
+
? new Set(allToMergeHashes)
|
|
18171
|
+
: await this.log.hasMany(allToMergeHashes);
|
|
18172
|
+
admittedMergeHashes = admittedHashes;
|
|
18173
|
+
const admittedShallowEntries =
|
|
18174
|
+
admittedHashes.size === allToMergeShallowEntries.length
|
|
18175
|
+
? allToMergeShallowEntries
|
|
18176
|
+
: allToMergeShallowEntries.filter((entry) =>
|
|
18177
|
+
admittedHashes.has(entry.hash),
|
|
18178
|
+
);
|
|
18179
|
+
if (!joinedPreparedFacts) {
|
|
18180
|
+
reusableCoordinatePersistItems =
|
|
18181
|
+
reusableCoordinatePersistItems.filter((item) =>
|
|
18182
|
+
admittedHashes.has(item.entry.hash),
|
|
18183
|
+
);
|
|
18184
|
+
coordinatePersistFallbackEntries =
|
|
18185
|
+
coordinatePersistFallbackEntries.filter((entry) =>
|
|
18186
|
+
admittedHashes.has(entry.hash),
|
|
18187
|
+
);
|
|
18188
|
+
}
|
|
18189
|
+
const reusableCoordinatePersistItemCount =
|
|
18190
|
+
reusableCoordinatePersistItems.length;
|
|
16504
18191
|
if (syncProfile) {
|
|
16505
18192
|
emitSyncProfileDuration(syncProfile, lowerLogJoinStartedAt, {
|
|
16506
18193
|
name: "sharedLog.receive.lowerLogJoin",
|
|
@@ -16512,6 +18199,7 @@ export class SharedLog<
|
|
|
16512
18199
|
batchHashOnlyEntryAdded,
|
|
16513
18200
|
programOnChange,
|
|
16514
18201
|
joinedPreparedFacts,
|
|
18202
|
+
admittedEntries: admittedHashes.size,
|
|
16515
18203
|
nativePreparedCoordinatesFinished,
|
|
16516
18204
|
},
|
|
16517
18205
|
});
|
|
@@ -16586,11 +18274,11 @@ export class SharedLog<
|
|
|
16586
18274
|
},
|
|
16587
18275
|
});
|
|
16588
18276
|
}
|
|
16589
|
-
for (const hash of
|
|
18277
|
+
for (const hash of admittedHashes) {
|
|
16590
18278
|
confirmedHashes.add(hash);
|
|
16591
18279
|
}
|
|
16592
18280
|
const checkedPruneStartedAt = syncProfileStart(syncProfile);
|
|
16593
|
-
await this.pruneJoinedEntriesNoLongerLed(
|
|
18281
|
+
await this.pruneJoinedEntriesNoLongerLed(admittedShallowEntries, {
|
|
16594
18282
|
decodedReplicaCounts: receiveReplicaCounts,
|
|
16595
18283
|
reusableLeaderPlans: reusableCoordinatePlans,
|
|
16596
18284
|
profile: syncProfile,
|
|
@@ -16605,15 +18293,17 @@ export class SharedLog<
|
|
|
16605
18293
|
}
|
|
16606
18294
|
|
|
16607
18295
|
for (const plan of joinPlans) {
|
|
16608
|
-
plan.toDelete
|
|
16609
|
-
|
|
16610
|
-
|
|
16611
|
-
|
|
16612
|
-
|
|
16613
|
-
|
|
16614
|
-
|
|
16615
|
-
|
|
16616
|
-
|
|
18296
|
+
plan.toDelete
|
|
18297
|
+
?.filter((entry) => admittedMergeHashes.has(entry.hash))
|
|
18298
|
+
.map((entry) =>
|
|
18299
|
+
this.pruneDebouncedFnAddIfNotKeeping({
|
|
18300
|
+
key: entry.hash,
|
|
18301
|
+
value: {
|
|
18302
|
+
entry,
|
|
18303
|
+
leaders: plan.leaders as Map<string, any>,
|
|
18304
|
+
},
|
|
18305
|
+
}),
|
|
18306
|
+
);
|
|
16617
18307
|
}
|
|
16618
18308
|
this.rebalanceParticipationDebounced?.call();
|
|
16619
18309
|
}
|
|
@@ -16623,17 +18313,23 @@ export class SharedLog<
|
|
|
16623
18313
|
continue;
|
|
16624
18314
|
}
|
|
16625
18315
|
for (const entries of plan.maybeDelete) {
|
|
18316
|
+
const admittedEntries = entries.filter((entry) =>
|
|
18317
|
+
admittedMergeHashes.has(getExchangeHeadHash(entry)),
|
|
18318
|
+
);
|
|
18319
|
+
if (admittedEntries.length === 0) {
|
|
18320
|
+
continue;
|
|
18321
|
+
}
|
|
16626
18322
|
const minReplicas = await this.getMaxReplicasFromHeads(
|
|
16627
|
-
this.getEntryGid(
|
|
18323
|
+
this.getEntryGid(admittedEntries[0].entry),
|
|
16628
18324
|
);
|
|
16629
18325
|
if (minReplicas != null) {
|
|
16630
18326
|
const isLeader = await this.isLeader({
|
|
16631
|
-
entry:
|
|
18327
|
+
entry: admittedEntries[0].entry,
|
|
16632
18328
|
replicas: minReplicas,
|
|
16633
18329
|
});
|
|
16634
18330
|
|
|
16635
18331
|
if (!isLeader) {
|
|
16636
|
-
for (const x of
|
|
18332
|
+
for (const x of admittedEntries) {
|
|
16637
18333
|
this.pruneDebouncedFnAddIfNotKeeping({
|
|
16638
18334
|
key: x.entry.hash,
|
|
16639
18335
|
value: {
|
|
@@ -16877,6 +18573,13 @@ export class SharedLog<
|
|
|
16877
18573
|
let pendingIHaveExtended = 0;
|
|
16878
18574
|
const leaderResponseHashes: string[] = [];
|
|
16879
18575
|
for (let i = 0; i < msg.hashes.length; i++) {
|
|
18576
|
+
if (
|
|
18577
|
+
!this.isReplicationLifecycleActive(
|
|
18578
|
+
receiveReplicationLifecycleController,
|
|
18579
|
+
)
|
|
18580
|
+
) {
|
|
18581
|
+
return;
|
|
18582
|
+
}
|
|
16880
18583
|
const hash = msg.hashes[i]!;
|
|
16881
18584
|
|
|
16882
18585
|
const nativeEntryGid = nativeLeaderHints.nativeEntryGids?.[i];
|
|
@@ -17012,8 +18715,21 @@ export class SharedLog<
|
|
|
17012
18715
|
resetTimeout: () => this.resetPendingIHaveTimeout(pendingIHave),
|
|
17013
18716
|
clear: () => this.clearPendingIHaveTimeout(pendingIHave),
|
|
17014
18717
|
callback: async (entry: Entry<T>) => {
|
|
17015
|
-
|
|
17016
|
-
|
|
18718
|
+
if (
|
|
18719
|
+
!this.isReplicationLifecycleActive(
|
|
18720
|
+
receiveReplicationLifecycleController,
|
|
18721
|
+
) ||
|
|
18722
|
+
requesting.size === 0
|
|
18723
|
+
) {
|
|
18724
|
+
return;
|
|
18725
|
+
}
|
|
18726
|
+
for (const requester of requesting) {
|
|
18727
|
+
this.removePeerFromGidPeerHistory(
|
|
18728
|
+
requester,
|
|
18729
|
+
entry.meta.gid,
|
|
18730
|
+
);
|
|
18731
|
+
this.removePruneRequestSent(entry.hash, requester);
|
|
18732
|
+
}
|
|
17017
18733
|
let isLeader = false;
|
|
17018
18734
|
await this._waitForEntryReplicators(
|
|
17019
18735
|
entry,
|
|
@@ -17032,12 +18748,19 @@ export class SharedLog<
|
|
|
17032
18748
|
},
|
|
17033
18749
|
},
|
|
17034
18750
|
);
|
|
18751
|
+
if (
|
|
18752
|
+
!this.isReplicationLifecycleActive(
|
|
18753
|
+
receiveReplicationLifecycleController,
|
|
18754
|
+
) ||
|
|
18755
|
+
requesting.size === 0
|
|
18756
|
+
) {
|
|
18757
|
+
return;
|
|
18758
|
+
}
|
|
17035
18759
|
if (isLeader) {
|
|
17036
18760
|
this.responseToPruneDebouncedFn.add({
|
|
17037
18761
|
hashes: [entry.hash],
|
|
17038
|
-
peers: requesting,
|
|
18762
|
+
peers: new Set(requesting),
|
|
17039
18763
|
});
|
|
17040
|
-
this._pendingIHave.delete(hash);
|
|
17041
18764
|
}
|
|
17042
18765
|
},
|
|
17043
18766
|
};
|
|
@@ -17080,19 +18803,32 @@ export class SharedLog<
|
|
|
17080
18803
|
}
|
|
17081
18804
|
} else if (msg instanceof ResponseIPrune) {
|
|
17082
18805
|
const lateResponses: string[] = [];
|
|
18806
|
+
const responseTasks: Promise<void>[] = [];
|
|
17083
18807
|
for (const hash of msg.hashes) {
|
|
17084
18808
|
const pendingDelete = this._checkedPrune.getPendingDelete(hash);
|
|
17085
18809
|
if (pendingDelete) {
|
|
17086
|
-
|
|
18810
|
+
responseTasks.push(
|
|
18811
|
+
Promise.resolve(
|
|
18812
|
+
pendingDelete.resolve(context.from.hashcode()),
|
|
18813
|
+
),
|
|
18814
|
+
);
|
|
17087
18815
|
} else {
|
|
17088
18816
|
lateResponses.push(hash);
|
|
17089
18817
|
}
|
|
17090
18818
|
}
|
|
17091
18819
|
if (lateResponses.length > 0) {
|
|
17092
|
-
|
|
17093
|
-
|
|
17094
|
-
|
|
17095
|
-
|
|
18820
|
+
responseTasks.push(
|
|
18821
|
+
this.recoverCheckedPruneFromLateResponses(
|
|
18822
|
+
lateResponses,
|
|
18823
|
+
context.from.hashcode(),
|
|
18824
|
+
),
|
|
18825
|
+
);
|
|
18826
|
+
}
|
|
18827
|
+
const results = await Promise.allSettled(responseTasks);
|
|
18828
|
+
for (const result of results) {
|
|
18829
|
+
if (result.status === "rejected") {
|
|
18830
|
+
logger.error(result.reason?.toString?.() ?? String(result.reason));
|
|
18831
|
+
}
|
|
17096
18832
|
}
|
|
17097
18833
|
} else if (msg instanceof ConfirmEntriesMessage) {
|
|
17098
18834
|
this.markEntriesKnownByPeer(msg.hashes, context.from.hashcode());
|
|
@@ -17100,10 +18836,25 @@ export class SharedLog<
|
|
|
17100
18836
|
return;
|
|
17101
18837
|
} else if (msg instanceof SyncCapabilitiesMessage) {
|
|
17102
18838
|
if (!context.from.equals(this.node.identity.publicKey)) {
|
|
17103
|
-
|
|
17104
|
-
|
|
17105
|
-
|
|
17106
|
-
|
|
18839
|
+
const openingEpoch =
|
|
18840
|
+
this._subscriptionOpeningEpochByPeer.get(receiveFromHash);
|
|
18841
|
+
if (
|
|
18842
|
+
this._replicationInfoBlockedPeers.has(receiveFromHash) &&
|
|
18843
|
+
openingEpoch === receiveSubscriptionEpoch
|
|
18844
|
+
) {
|
|
18845
|
+
// A prior unsubscribe cleanup may still be ahead of this reconnect
|
|
18846
|
+
// barrier. Stage the new generation's one-shot advertisement so that
|
|
18847
|
+
// cleanup cannot erase it before the opening transition commits.
|
|
18848
|
+
this._openingSyncCapabilitiesByPeer.set(receiveFromHash, {
|
|
18849
|
+
epoch: openingEpoch,
|
|
18850
|
+
capabilities: msg.capabilities,
|
|
18851
|
+
});
|
|
18852
|
+
} else {
|
|
18853
|
+
this._peerSyncCapabilities.set(
|
|
18854
|
+
receiveFromHash,
|
|
18855
|
+
msg.capabilities,
|
|
18856
|
+
);
|
|
18857
|
+
}
|
|
17107
18858
|
}
|
|
17108
18859
|
return;
|
|
17109
18860
|
} else if (await this.syncronizer.onMessage(msg, context)) {
|
|
@@ -17120,10 +18871,14 @@ export class SharedLog<
|
|
|
17120
18871
|
return;
|
|
17121
18872
|
}
|
|
17122
18873
|
const replicationLifecycleController =
|
|
17123
|
-
|
|
18874
|
+
receiveReplicationLifecycleController;
|
|
17124
18875
|
if (
|
|
17125
18876
|
!replicationLifecycleController ||
|
|
17126
|
-
!this.
|
|
18877
|
+
!this.isPeerReceiveAdmissionOpen(
|
|
18878
|
+
receiveFromHash,
|
|
18879
|
+
replicationLifecycleController,
|
|
18880
|
+
receiveSubscriptionEpoch,
|
|
18881
|
+
)
|
|
17127
18882
|
) {
|
|
17128
18883
|
return;
|
|
17129
18884
|
}
|
|
@@ -17133,8 +18888,10 @@ export class SharedLog<
|
|
|
17133
18888
|
replicationSegments = await this.getMyReplicationSegments();
|
|
17134
18889
|
} catch (error) {
|
|
17135
18890
|
if (
|
|
17136
|
-
!this.
|
|
18891
|
+
!this.isPeerReceiveAdmissionOpen(
|
|
18892
|
+
receiveFromHash,
|
|
17137
18893
|
replicationLifecycleController,
|
|
18894
|
+
receiveSubscriptionEpoch,
|
|
17138
18895
|
) &&
|
|
17139
18896
|
isNotStartedError(error as Error)
|
|
17140
18897
|
) {
|
|
@@ -17143,7 +18900,11 @@ export class SharedLog<
|
|
|
17143
18900
|
throw error;
|
|
17144
18901
|
}
|
|
17145
18902
|
if (
|
|
17146
|
-
!this.
|
|
18903
|
+
!this.isPeerReceiveAdmissionOpen(
|
|
18904
|
+
receiveFromHash,
|
|
18905
|
+
replicationLifecycleController,
|
|
18906
|
+
receiveSubscriptionEpoch,
|
|
18907
|
+
)
|
|
17147
18908
|
) {
|
|
17148
18909
|
return;
|
|
17149
18910
|
}
|
|
@@ -17164,7 +18925,11 @@ export class SharedLog<
|
|
|
17164
18925
|
),
|
|
17165
18926
|
);
|
|
17166
18927
|
if (
|
|
17167
|
-
!this.
|
|
18928
|
+
!this.isPeerReceiveAdmissionOpen(
|
|
18929
|
+
receiveFromHash,
|
|
18930
|
+
replicationLifecycleController,
|
|
18931
|
+
receiveSubscriptionEpoch,
|
|
18932
|
+
)
|
|
17168
18933
|
) {
|
|
17169
18934
|
return;
|
|
17170
18935
|
}
|
|
@@ -17214,14 +18979,38 @@ export class SharedLog<
|
|
|
17214
18979
|
// (and downstream `waitForReplicator()` timeouts) under timing-sensitive joins.
|
|
17215
18980
|
const from = context.from!;
|
|
17216
18981
|
const fromHash = from.hashcode();
|
|
17217
|
-
if (
|
|
18982
|
+
if (
|
|
18983
|
+
!this.isReplicationLifecycleActive(
|
|
18984
|
+
receiveReplicationLifecycleController,
|
|
18985
|
+
) ||
|
|
18986
|
+
!this.isCurrentReplicationInfoReceiveEpoch(
|
|
18987
|
+
receiveFromHash,
|
|
18988
|
+
receiveReplicationInfoReceiveEpoch,
|
|
18989
|
+
) ||
|
|
18990
|
+
this._replicationInfoBlockedPeers.has(fromHash)
|
|
18991
|
+
) {
|
|
17218
18992
|
return;
|
|
17219
18993
|
}
|
|
17220
18994
|
const messageTimestamp = context.message.header.timestamp;
|
|
18995
|
+
releasePeerReceiveLease?.();
|
|
18996
|
+
releasePeerReceiveLease = undefined;
|
|
17221
18997
|
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
17222
18998
|
try {
|
|
17223
18999
|
// The peer may have unsubscribed after this message was queued.
|
|
17224
|
-
if (
|
|
19000
|
+
if (
|
|
19001
|
+
!this.isReplicationLifecycleActive(
|
|
19002
|
+
receiveReplicationLifecycleController,
|
|
19003
|
+
) ||
|
|
19004
|
+
!this.isCurrentSubscriptionEpoch(
|
|
19005
|
+
fromHash,
|
|
19006
|
+
receiveSubscriptionEpoch,
|
|
19007
|
+
) ||
|
|
19008
|
+
!this.isCurrentReplicationInfoReceiveEpoch(
|
|
19009
|
+
fromHash,
|
|
19010
|
+
receiveReplicationInfoReceiveEpoch,
|
|
19011
|
+
) ||
|
|
19012
|
+
this._replicationInfoBlockedPeers.has(fromHash)
|
|
19013
|
+
) {
|
|
17225
19014
|
return;
|
|
17226
19015
|
}
|
|
17227
19016
|
|
|
@@ -17269,29 +19058,72 @@ export class SharedLog<
|
|
|
17269
19058
|
}
|
|
17270
19059
|
});
|
|
17271
19060
|
} else if (msg instanceof StoppedReplicating) {
|
|
17272
|
-
|
|
19061
|
+
const from = context.from!;
|
|
19062
|
+
const segmentIds = msg.segmentIds;
|
|
19063
|
+
if (from.equals(this.node.identity.publicKey)) {
|
|
17273
19064
|
return;
|
|
17274
19065
|
}
|
|
17275
|
-
const fromHash =
|
|
17276
|
-
if (
|
|
19066
|
+
const fromHash = from.hashcode();
|
|
19067
|
+
if (
|
|
19068
|
+
!this.isReplicationLifecycleActive(
|
|
19069
|
+
receiveReplicationLifecycleController,
|
|
19070
|
+
) ||
|
|
19071
|
+
!this.isCurrentReplicationInfoReceiveEpoch(
|
|
19072
|
+
receiveFromHash,
|
|
19073
|
+
receiveReplicationInfoReceiveEpoch,
|
|
19074
|
+
) ||
|
|
19075
|
+
this._replicationInfoBlockedPeers.has(fromHash)
|
|
19076
|
+
) {
|
|
17277
19077
|
return;
|
|
17278
19078
|
}
|
|
17279
|
-
|
|
19079
|
+
const messageTimestamp = context.message.header.timestamp;
|
|
19080
|
+
releasePeerReceiveLease?.();
|
|
19081
|
+
releasePeerReceiveLease = undefined;
|
|
19082
|
+
await this.withReplicationInfoApplyQueue(fromHash, async () => {
|
|
19083
|
+
if (
|
|
19084
|
+
!this.isReplicationLifecycleActive(
|
|
19085
|
+
receiveReplicationLifecycleController,
|
|
19086
|
+
) ||
|
|
19087
|
+
!this.isCurrentSubscriptionEpoch(
|
|
19088
|
+
fromHash,
|
|
19089
|
+
receiveSubscriptionEpoch,
|
|
19090
|
+
) ||
|
|
19091
|
+
!this.isCurrentReplicationInfoReceiveEpoch(
|
|
19092
|
+
fromHash,
|
|
19093
|
+
receiveReplicationInfoReceiveEpoch,
|
|
19094
|
+
) ||
|
|
19095
|
+
this._replicationInfoBlockedPeers.has(fromHash)
|
|
19096
|
+
) {
|
|
19097
|
+
return;
|
|
19098
|
+
}
|
|
17280
19099
|
|
|
17281
|
-
|
|
17282
|
-
|
|
17283
|
-
|
|
17284
|
-
|
|
19100
|
+
const previousTimestamp =
|
|
19101
|
+
this.latestReplicationInfoMessage.get(fromHash);
|
|
19102
|
+
if (previousTimestamp && previousTimestamp > messageTimestamp) {
|
|
19103
|
+
return;
|
|
19104
|
+
}
|
|
19105
|
+
this.latestReplicationInfoMessage.set(fromHash, messageTimestamp);
|
|
19106
|
+
this._replicatorLivenessFailures.delete(fromHash);
|
|
19107
|
+
if (this.closed) {
|
|
19108
|
+
return;
|
|
19109
|
+
}
|
|
17285
19110
|
|
|
17286
|
-
|
|
17287
|
-
|
|
17288
|
-
|
|
17289
|
-
|
|
17290
|
-
|
|
17291
|
-
|
|
17292
|
-
|
|
17293
|
-
|
|
17294
|
-
|
|
19111
|
+
const rangesToRemove =
|
|
19112
|
+
await this.resolveReplicationRangesFromIdsAndKey(
|
|
19113
|
+
segmentIds,
|
|
19114
|
+
from,
|
|
19115
|
+
);
|
|
19116
|
+
|
|
19117
|
+
await this.removeReplicationRanges(rangesToRemove, from);
|
|
19118
|
+
const timestamp = BigInt(+new Date());
|
|
19119
|
+
for (const range of rangesToRemove) {
|
|
19120
|
+
this.replicationChangeDebounceFn.add({
|
|
19121
|
+
range,
|
|
19122
|
+
type: "removed",
|
|
19123
|
+
timestamp,
|
|
19124
|
+
});
|
|
19125
|
+
}
|
|
19126
|
+
});
|
|
17295
19127
|
} else {
|
|
17296
19128
|
throw new Error("Unexpected message");
|
|
17297
19129
|
}
|
|
@@ -17345,6 +19177,8 @@ export class SharedLog<
|
|
|
17345
19177
|
// Every return and every locally swallowed receive error passes this
|
|
17346
19178
|
// boundary. Release a native wire stash exactly once first, then surface
|
|
17347
19179
|
// any durable mutation poison that arose while handling the message.
|
|
19180
|
+
releasePeerReceiveLease?.();
|
|
19181
|
+
releasePeerReceiveLease = undefined;
|
|
17348
19182
|
this.throwIfNativeDurableCommitFailed();
|
|
17349
19183
|
}
|
|
17350
19184
|
}
|
|
@@ -17987,30 +19821,35 @@ export class SharedLog<
|
|
|
17987
19821
|
checkLeaders: (options: WaitForReplicatorsOptions<R>) => Promise<LeaderMap>,
|
|
17988
19822
|
): Promise<LeaderMap | false> {
|
|
17989
19823
|
const timeout = options.timeout ?? this.waitForReplicatorTimeout;
|
|
19824
|
+
const closeSignal = this._closeController.signal;
|
|
19825
|
+
const replicationLifecycleSignal =
|
|
19826
|
+
this._replicationLifecycleController?.signal;
|
|
17990
19827
|
|
|
17991
19828
|
return new Promise((resolve, reject) => {
|
|
17992
19829
|
let settled = false;
|
|
19830
|
+
const checks = new Set<Promise<void>>();
|
|
17993
19831
|
const removeListeners = () => {
|
|
17994
19832
|
this.events.removeEventListener("replication:change", roleListener);
|
|
17995
19833
|
this.events.removeEventListener("replicator:mature", roleListener);
|
|
17996
|
-
|
|
17997
|
-
|
|
17998
|
-
abortListener,
|
|
17999
|
-
);
|
|
19834
|
+
closeSignal.removeEventListener("abort", abortListener);
|
|
19835
|
+
replicationLifecycleSignal?.removeEventListener("abort", abortListener);
|
|
18000
19836
|
};
|
|
18001
19837
|
const settleResolve = (value: LeaderMap | false) => {
|
|
18002
19838
|
if (settled) return;
|
|
18003
19839
|
settled = true;
|
|
18004
19840
|
removeListeners();
|
|
18005
19841
|
clearTimeout(timer);
|
|
18006
|
-
|
|
19842
|
+
// Leader planning may persist coordinates. Keep the caller (and any
|
|
19843
|
+
// receive lease it owns) alive until checks admitted before this
|
|
19844
|
+
// timeout/abort have finished their local side effects.
|
|
19845
|
+
void Promise.allSettled([...checks]).then(() => resolve(value));
|
|
18007
19846
|
};
|
|
18008
19847
|
const settleReject = (error: unknown) => {
|
|
18009
19848
|
if (settled) return;
|
|
18010
19849
|
settled = true;
|
|
18011
19850
|
removeListeners();
|
|
18012
19851
|
clearTimeout(timer);
|
|
18013
|
-
reject(error);
|
|
19852
|
+
void Promise.allSettled([...checks]).then(() => reject(error));
|
|
18014
19853
|
};
|
|
18015
19854
|
const abortListener = () => {
|
|
18016
19855
|
settleResolve(false);
|
|
@@ -18044,9 +19883,14 @@ export class SharedLog<
|
|
|
18044
19883
|
settleResolve(leaders);
|
|
18045
19884
|
};
|
|
18046
19885
|
const runCheck = () => {
|
|
18047
|
-
|
|
18048
|
-
|
|
18049
|
-
|
|
19886
|
+
if (settled) return;
|
|
19887
|
+
let running!: Promise<void>;
|
|
19888
|
+
running = check()
|
|
19889
|
+
.catch((error) => {
|
|
19890
|
+
settleReject(error);
|
|
19891
|
+
})
|
|
19892
|
+
.finally(() => checks.delete(running));
|
|
19893
|
+
checks.add(running);
|
|
18050
19894
|
};
|
|
18051
19895
|
|
|
18052
19896
|
const roleListener = () => {
|
|
@@ -18055,7 +19899,15 @@ export class SharedLog<
|
|
|
18055
19899
|
|
|
18056
19900
|
this.events.addEventListener("replication:change", roleListener);
|
|
18057
19901
|
this.events.addEventListener("replicator:mature", roleListener);
|
|
18058
|
-
|
|
19902
|
+
closeSignal.addEventListener("abort", abortListener);
|
|
19903
|
+
replicationLifecycleSignal?.addEventListener("abort", abortListener);
|
|
19904
|
+
// AbortSignal does not replay an abort event to listeners added after it
|
|
19905
|
+
// fired. Recheck after registration so work started concurrently with the
|
|
19906
|
+
// terminal fence cannot wait for the full leader-selection timeout.
|
|
19907
|
+
if (closeSignal.aborted || replicationLifecycleSignal?.aborted) {
|
|
19908
|
+
abortListener();
|
|
19909
|
+
return;
|
|
19910
|
+
}
|
|
18059
19911
|
runCheck();
|
|
18060
19912
|
});
|
|
18061
19913
|
}
|
|
@@ -21059,6 +22911,53 @@ export class SharedLog<
|
|
|
21059
22911
|
});
|
|
21060
22912
|
}
|
|
21061
22913
|
|
|
22914
|
+
private async drainReplicationInfoApplyQueues(): Promise<void> {
|
|
22915
|
+
for (;;) {
|
|
22916
|
+
const tails = [...(this._replicationInfoApplyQueueByPeer?.values() ?? [])];
|
|
22917
|
+
if (tails.length === 0) {
|
|
22918
|
+
return;
|
|
22919
|
+
}
|
|
22920
|
+
await Promise.allSettled(tails);
|
|
22921
|
+
// Queue cleanup runs in `finally`; give it a microtask before checking for
|
|
22922
|
+
// tails admitted while the previous snapshot was settling.
|
|
22923
|
+
await Promise.resolve();
|
|
22924
|
+
}
|
|
22925
|
+
}
|
|
22926
|
+
|
|
22927
|
+
private advanceReplicationInfoReceiveEpoch(peerHash: string): object {
|
|
22928
|
+
const next = {};
|
|
22929
|
+
this._replicationInfoReceiveEpochByPeer.set(peerHash, next);
|
|
22930
|
+
return next;
|
|
22931
|
+
}
|
|
22932
|
+
|
|
22933
|
+
private getReplicationInfoReceiveEpoch(peerHash: string): object | null {
|
|
22934
|
+
return this._replicationInfoReceiveEpochByPeer.get(peerHash) ?? null;
|
|
22935
|
+
}
|
|
22936
|
+
|
|
22937
|
+
private isCurrentReplicationInfoReceiveEpoch(
|
|
22938
|
+
peerHash: string,
|
|
22939
|
+
epoch: object | null,
|
|
22940
|
+
): boolean {
|
|
22941
|
+
return this.getReplicationInfoReceiveEpoch(peerHash) === epoch;
|
|
22942
|
+
}
|
|
22943
|
+
|
|
22944
|
+
private advanceSubscriptionEpoch(peerHash: string): object {
|
|
22945
|
+
const next = {};
|
|
22946
|
+
this._subscriptionEpochByPeer.set(peerHash, next);
|
|
22947
|
+
return next;
|
|
22948
|
+
}
|
|
22949
|
+
|
|
22950
|
+
private getSubscriptionEpoch(peerHash: string): object | null {
|
|
22951
|
+
return this._subscriptionEpochByPeer.get(peerHash) ?? null;
|
|
22952
|
+
}
|
|
22953
|
+
|
|
22954
|
+
private isCurrentSubscriptionEpoch(
|
|
22955
|
+
peerHash: string,
|
|
22956
|
+
epoch: object | null,
|
|
22957
|
+
): boolean {
|
|
22958
|
+
return this.getSubscriptionEpoch(peerHash) === epoch;
|
|
22959
|
+
}
|
|
22960
|
+
|
|
21062
22961
|
private cancelReplicationInfoRequests(peerHash: string) {
|
|
21063
22962
|
const state = this._replicationInfoRequestByPeer.get(peerHash);
|
|
21064
22963
|
if (!state) return;
|
|
@@ -21147,6 +23046,7 @@ export class SharedLog<
|
|
|
21147
23046
|
publicKey: PublicSignKey,
|
|
21148
23047
|
topics: string[],
|
|
21149
23048
|
subscribed: boolean,
|
|
23049
|
+
subscriptionEpoch?: object,
|
|
21150
23050
|
) {
|
|
21151
23051
|
if (!topics.includes(this.topic)) {
|
|
21152
23052
|
return;
|
|
@@ -21160,8 +23060,76 @@ export class SharedLog<
|
|
|
21160
23060
|
}
|
|
21161
23061
|
|
|
21162
23062
|
const peerHash = publicKey.hashcode();
|
|
23063
|
+
const expectedSubscriptionEpoch =
|
|
23064
|
+
subscriptionEpoch ?? this.advanceSubscriptionEpoch(peerHash);
|
|
23065
|
+
const ownsSubscriptionEpoch = () =>
|
|
23066
|
+
this.isCurrentSubscriptionEpoch(peerHash, expectedSubscriptionEpoch);
|
|
23067
|
+
if (!ownsSubscriptionEpoch()) {
|
|
23068
|
+
return;
|
|
23069
|
+
}
|
|
23070
|
+
if (subscribed) {
|
|
23071
|
+
const pendingOpeningCapabilities =
|
|
23072
|
+
this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
23073
|
+
if (
|
|
23074
|
+
pendingOpeningCapabilities &&
|
|
23075
|
+
pendingOpeningCapabilities.epoch !== expectedSubscriptionEpoch
|
|
23076
|
+
) {
|
|
23077
|
+
this._openingSyncCapabilitiesByPeer.delete(peerHash);
|
|
23078
|
+
}
|
|
23079
|
+
this._subscriptionOpeningEpochByPeer.set(
|
|
23080
|
+
peerHash,
|
|
23081
|
+
expectedSubscriptionEpoch,
|
|
23082
|
+
);
|
|
23083
|
+
// Fence new messages immediately, drain handlers admitted by the previous
|
|
23084
|
+
// subscription, then wait behind every queued replication mutation. A
|
|
23085
|
+
// reconnect must not inherit metadata or ranges from the old connection.
|
|
23086
|
+
try {
|
|
23087
|
+
this._replicationInfoBlockedPeers.add(peerHash);
|
|
23088
|
+
await this.drainPeerReceiveHandlers(peerHash);
|
|
23089
|
+
await this.withReplicationInfoApplyQueue(peerHash, async () => {});
|
|
23090
|
+
if (
|
|
23091
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
23092
|
+
!ownsSubscriptionEpoch()
|
|
23093
|
+
) {
|
|
23094
|
+
return;
|
|
23095
|
+
}
|
|
23096
|
+
// The timestamp watermark belongs to the previous subscription epoch.
|
|
23097
|
+
// Sender clocks are not synchronized, so carrying a local unsubscribe
|
|
23098
|
+
// timestamp forward could reject every valid announcement after reconnect.
|
|
23099
|
+
this.latestReplicationInfoMessage.delete(peerHash);
|
|
23100
|
+
this._pendingReplicatorLeaveByPeer.delete(peerHash);
|
|
23101
|
+
const openingCapabilities =
|
|
23102
|
+
this._openingSyncCapabilitiesByPeer.get(peerHash);
|
|
23103
|
+
if (openingCapabilities?.epoch === expectedSubscriptionEpoch) {
|
|
23104
|
+
this._peerSyncCapabilities.set(
|
|
23105
|
+
peerHash,
|
|
23106
|
+
openingCapabilities.capabilities,
|
|
23107
|
+
);
|
|
23108
|
+
this._openingSyncCapabilitiesByPeer.delete(peerHash);
|
|
23109
|
+
}
|
|
23110
|
+
this._replicationInfoBlockedPeers.delete(peerHash);
|
|
23111
|
+
} finally {
|
|
23112
|
+
if (
|
|
23113
|
+
this._openingSyncCapabilitiesByPeer.get(peerHash)?.epoch ===
|
|
23114
|
+
expectedSubscriptionEpoch
|
|
23115
|
+
) {
|
|
23116
|
+
this._openingSyncCapabilitiesByPeer.delete(peerHash);
|
|
23117
|
+
}
|
|
23118
|
+
if (
|
|
23119
|
+
this._subscriptionOpeningEpochByPeer.get(peerHash) ===
|
|
23120
|
+
expectedSubscriptionEpoch
|
|
23121
|
+
) {
|
|
23122
|
+
this._subscriptionOpeningEpochByPeer.delete(peerHash);
|
|
23123
|
+
}
|
|
23124
|
+
}
|
|
23125
|
+
}
|
|
21163
23126
|
if (!subscribed) {
|
|
23127
|
+
this._subscriptionOpeningEpochByPeer.delete(peerHash);
|
|
23128
|
+
this._openingSyncCapabilitiesByPeer.delete(peerHash);
|
|
21164
23129
|
this._replicationInfoBlockedPeers.add(peerHash);
|
|
23130
|
+
const disconnectedJoinWarmupGeneration =
|
|
23131
|
+
this._joinWarmupGenerationByTarget.get(peerHash) ?? null;
|
|
23132
|
+
this.cancelJoinWarmupTarget(peerHash);
|
|
21165
23133
|
|
|
21166
23134
|
const now = BigInt(+new Date());
|
|
21167
23135
|
const previous = this.latestReplicationInfoMessage.get(peerHash);
|
|
@@ -21169,24 +23137,37 @@ export class SharedLog<
|
|
|
21169
23137
|
this.latestReplicationInfoMessage.set(peerHash, now);
|
|
21170
23138
|
}
|
|
21171
23139
|
|
|
21172
|
-
|
|
23140
|
+
let removed = false;
|
|
21173
23141
|
try {
|
|
21174
23142
|
// Unsubscribe can race with the peer's final replication reset message.
|
|
21175
23143
|
// Proactively evict its ranges so leader selection doesn't keep stale owners.
|
|
21176
|
-
await this.removeReplicator(publicKey, {
|
|
23144
|
+
removed = await this.removeReplicator(publicKey, {
|
|
23145
|
+
cleanupIfSubscriptionSuperseded: true,
|
|
23146
|
+
expectedJoinWarmupGeneration:
|
|
23147
|
+
disconnectedJoinWarmupGeneration,
|
|
23148
|
+
noEvent: true,
|
|
23149
|
+
onRemoved: ({ wasReplicator }) => {
|
|
23150
|
+
if (wasReplicator) {
|
|
23151
|
+
this._pendingReplicatorLeaveByPeer.add(peerHash);
|
|
23152
|
+
}
|
|
23153
|
+
},
|
|
23154
|
+
replicationLifecycleController,
|
|
23155
|
+
subscriptionEpoch: expectedSubscriptionEpoch,
|
|
23156
|
+
});
|
|
21177
23157
|
} catch (error) {
|
|
21178
23158
|
if (!isNotStartedError(error as Error)) {
|
|
21179
23159
|
throw error;
|
|
21180
23160
|
}
|
|
21181
23161
|
}
|
|
21182
|
-
if (
|
|
23162
|
+
if (
|
|
23163
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
23164
|
+
!ownsSubscriptionEpoch() ||
|
|
23165
|
+
!removed
|
|
23166
|
+
) {
|
|
21183
23167
|
return;
|
|
21184
23168
|
}
|
|
21185
23169
|
|
|
21186
|
-
this.
|
|
21187
|
-
this.cleanupPeerDisconnectTracking(peerHash);
|
|
21188
|
-
|
|
21189
|
-
if (wasReplicator) {
|
|
23170
|
+
if (this._pendingReplicatorLeaveByPeer.delete(peerHash)) {
|
|
21190
23171
|
this.events.dispatchEvent(
|
|
21191
23172
|
new CustomEvent<ReplicatorLeaveEvent>("replicator:leave", {
|
|
21192
23173
|
detail: { publicKey },
|
|
@@ -21234,7 +23215,10 @@ export class SharedLog<
|
|
|
21234
23215
|
}
|
|
21235
23216
|
throw error;
|
|
21236
23217
|
}
|
|
21237
|
-
if (
|
|
23218
|
+
if (
|
|
23219
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
23220
|
+
!ownsSubscriptionEpoch()
|
|
23221
|
+
) {
|
|
21238
23222
|
return;
|
|
21239
23223
|
}
|
|
21240
23224
|
if (replicationSegments.length > 0) {
|
|
@@ -21254,7 +23238,10 @@ export class SharedLog<
|
|
|
21254
23238
|
replicationLifecycleController,
|
|
21255
23239
|
),
|
|
21256
23240
|
);
|
|
21257
|
-
if (
|
|
23241
|
+
if (
|
|
23242
|
+
!this.isReplicationLifecycleActive(replicationLifecycleController) ||
|
|
23243
|
+
!ownsSubscriptionEpoch()
|
|
23244
|
+
) {
|
|
21258
23245
|
return;
|
|
21259
23246
|
}
|
|
21260
23247
|
|
|
@@ -21285,7 +23272,10 @@ export class SharedLog<
|
|
|
21285
23272
|
// Request the remote peer's replication info. This makes joins resilient to
|
|
21286
23273
|
// timing-sensitive delivery/order issues where we may miss their initial
|
|
21287
23274
|
// replication announcement.
|
|
21288
|
-
if (
|
|
23275
|
+
if (
|
|
23276
|
+
this.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
23277
|
+
ownsSubscriptionEpoch()
|
|
23278
|
+
) {
|
|
21289
23279
|
this.scheduleReplicationInfoRequests(
|
|
21290
23280
|
publicKey,
|
|
21291
23281
|
replicationLifecycleController,
|
|
@@ -21718,14 +23708,23 @@ export class SharedLog<
|
|
|
21718
23708
|
if (this.closed) {
|
|
21719
23709
|
return;
|
|
21720
23710
|
}
|
|
21721
|
-
|
|
21722
|
-
await this.log.trim();
|
|
21723
|
-
|
|
21724
23711
|
const batchedChanges = Array.isArray(changeOrChanges[0])
|
|
21725
23712
|
? (changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>[])
|
|
21726
23713
|
: [changeOrChanges as ReplicationChanges<ReplicationRangeIndexable<R>>];
|
|
21727
23714
|
const changes = batchedChanges.flat();
|
|
21728
23715
|
const selfHash = this.node.identity.publicKey.hashcode();
|
|
23716
|
+
const joinWarmupGenerations = new Map<string, object>();
|
|
23717
|
+
for (const change of changes) {
|
|
23718
|
+
if (change.type === "added" && change.range.hash !== selfHash) {
|
|
23719
|
+
joinWarmupGenerations.set(
|
|
23720
|
+
change.range.hash,
|
|
23721
|
+
this.getJoinWarmupGeneration(change.range.hash),
|
|
23722
|
+
);
|
|
23723
|
+
}
|
|
23724
|
+
}
|
|
23725
|
+
|
|
23726
|
+
await this.log.trim();
|
|
23727
|
+
|
|
21729
23728
|
// On removed ranges (peer leaves / shrink), gid-level history can hide
|
|
21730
23729
|
// per-entry gaps. Force a fresh delivery pass for reassigned entries.
|
|
21731
23730
|
const forceFreshDelivery = changes.some(
|
|
@@ -21803,6 +23802,12 @@ export class SharedLog<
|
|
|
21803
23802
|
),
|
|
21804
23803
|
)
|
|
21805
23804
|
: changes;
|
|
23805
|
+
const isCurrentJoinWarmupTarget = (target: string) =>
|
|
23806
|
+
warmupPeers.has(target) &&
|
|
23807
|
+
this._joinWarmupGenerationByTarget.get(target) ===
|
|
23808
|
+
joinWarmupGenerations.get(target);
|
|
23809
|
+
const areJoinWarmupGenerationsCurrent = () =>
|
|
23810
|
+
[...warmupPeers].every(isCurrentJoinWarmupTarget);
|
|
21806
23811
|
|
|
21807
23812
|
try {
|
|
21808
23813
|
const uncheckedDeliver: Map<
|
|
@@ -21815,6 +23820,10 @@ export class SharedLog<
|
|
|
21815
23820
|
return;
|
|
21816
23821
|
}
|
|
21817
23822
|
const isWarmupTarget = warmupPeers.has(target);
|
|
23823
|
+
if (isWarmupTarget && !isCurrentJoinWarmupTarget(target)) {
|
|
23824
|
+
uncheckedDeliver.delete(target);
|
|
23825
|
+
return;
|
|
23826
|
+
}
|
|
21818
23827
|
const mode: RepairDispatchMode = forceFreshDelivery
|
|
21819
23828
|
? "churn"
|
|
21820
23829
|
: isWarmupTarget
|
|
@@ -21839,6 +23848,9 @@ export class SharedLog<
|
|
|
21839
23848
|
target: string,
|
|
21840
23849
|
entry: EntryReplicated<any>,
|
|
21841
23850
|
) => {
|
|
23851
|
+
if (warmupPeers.has(target) && !isCurrentJoinWarmupTarget(target)) {
|
|
23852
|
+
return;
|
|
23853
|
+
}
|
|
21842
23854
|
churnRepairPeers.add(target);
|
|
21843
23855
|
let set = uncheckedDeliver.get(target);
|
|
21844
23856
|
if (!set) {
|
|
@@ -21863,7 +23875,11 @@ export class SharedLog<
|
|
|
21863
23875
|
forceFresh: forceFreshDelivery || useJoinWarmupFastPath,
|
|
21864
23876
|
},
|
|
21865
23877
|
)) {
|
|
21866
|
-
if (
|
|
23878
|
+
if (
|
|
23879
|
+
this.closed ||
|
|
23880
|
+
(useJoinWarmupFastPath &&
|
|
23881
|
+
!areJoinWarmupGenerationsCurrent())
|
|
23882
|
+
) {
|
|
21867
23883
|
break;
|
|
21868
23884
|
}
|
|
21869
23885
|
|
|
@@ -21883,7 +23899,9 @@ export class SharedLog<
|
|
|
21883
23899
|
|
|
21884
23900
|
const candidatePeers = new Set<string>([selfHash]);
|
|
21885
23901
|
for (const target of warmupPeers) {
|
|
21886
|
-
|
|
23902
|
+
if (isCurrentJoinWarmupTarget(target)) {
|
|
23903
|
+
candidatePeers.add(target);
|
|
23904
|
+
}
|
|
21887
23905
|
}
|
|
21888
23906
|
if (oldPeersSet) {
|
|
21889
23907
|
for (const oldPeer of oldPeersSet) {
|
|
@@ -21900,6 +23918,9 @@ export class SharedLog<
|
|
|
21900
23918
|
persist: false,
|
|
21901
23919
|
},
|
|
21902
23920
|
);
|
|
23921
|
+
if (!areJoinWarmupGenerationsCurrent()) {
|
|
23922
|
+
continue;
|
|
23923
|
+
}
|
|
21903
23924
|
|
|
21904
23925
|
if (oldPeersSet) {
|
|
21905
23926
|
for (const oldPeer of oldPeersSet) {
|
|
@@ -21910,14 +23931,18 @@ export class SharedLog<
|
|
|
21910
23931
|
}
|
|
21911
23932
|
|
|
21912
23933
|
for (const [peer] of currentPeers) {
|
|
21913
|
-
if (
|
|
21914
|
-
this.markRepairSweepOptimisticPeer(
|
|
23934
|
+
if (isCurrentJoinWarmupTarget(peer)) {
|
|
23935
|
+
this.markRepairSweepOptimisticPeer(
|
|
23936
|
+
entryReplicated.gid,
|
|
23937
|
+
peer,
|
|
23938
|
+
joinWarmupGenerations.get(peer)!,
|
|
23939
|
+
);
|
|
21915
23940
|
}
|
|
21916
23941
|
}
|
|
21917
23942
|
|
|
21918
23943
|
const authoritativePeers = [...currentPeers.keys()].filter(
|
|
21919
23944
|
(peer) =>
|
|
21920
|
-
!
|
|
23945
|
+
!isCurrentJoinWarmupTarget(peer) &&
|
|
21921
23946
|
!this.hasPendingRepairSweepOptimisticPeer(
|
|
21922
23947
|
entryReplicated.gid,
|
|
21923
23948
|
peer,
|
|
@@ -21983,8 +24008,12 @@ export class SharedLog<
|
|
|
21983
24008
|
}
|
|
21984
24009
|
|
|
21985
24010
|
for (const [peer] of currentPeers) {
|
|
21986
|
-
if (
|
|
21987
|
-
this.markRepairSweepOptimisticPeer(
|
|
24011
|
+
if (isCurrentJoinWarmupTarget(peer)) {
|
|
24012
|
+
this.markRepairSweepOptimisticPeer(
|
|
24013
|
+
entryReplicated.gid,
|
|
24014
|
+
peer,
|
|
24015
|
+
joinWarmupGenerations.get(peer)!,
|
|
24016
|
+
);
|
|
21988
24017
|
}
|
|
21989
24018
|
}
|
|
21990
24019
|
|
|
@@ -22036,6 +24065,7 @@ export class SharedLog<
|
|
|
22036
24065
|
this.scheduleRepairSweep({
|
|
22037
24066
|
mode: "join-warmup",
|
|
22038
24067
|
peers,
|
|
24068
|
+
joinWarmupGenerations,
|
|
22039
24069
|
});
|
|
22040
24070
|
}, 250);
|
|
22041
24071
|
timer.unref?.();
|
|
@@ -22110,6 +24140,7 @@ export class SharedLog<
|
|
|
22110
24140
|
}
|
|
22111
24141
|
|
|
22112
24142
|
const fromHash = evt.detail.from.hashcode();
|
|
24143
|
+
const subscriptionEpoch = this.advanceSubscriptionEpoch(fromHash);
|
|
22113
24144
|
this._replicationInfoBlockedPeers.add(fromHash);
|
|
22114
24145
|
this._recentRepairDispatch.delete(fromHash);
|
|
22115
24146
|
|
|
@@ -22127,6 +24158,7 @@ export class SharedLog<
|
|
|
22127
24158
|
evt.detail.from,
|
|
22128
24159
|
evt.detail.topics,
|
|
22129
24160
|
false,
|
|
24161
|
+
subscriptionEpoch,
|
|
22130
24162
|
);
|
|
22131
24163
|
}
|
|
22132
24164
|
|
|
@@ -22140,14 +24172,17 @@ export class SharedLog<
|
|
|
22140
24172
|
return;
|
|
22141
24173
|
}
|
|
22142
24174
|
|
|
24175
|
+
const fromHash = evt.detail.from.hashcode();
|
|
24176
|
+
const subscriptionEpoch = this.advanceSubscriptionEpoch(fromHash);
|
|
22143
24177
|
this.remoteBlocks.onReachable(evt.detail.from);
|
|
22144
|
-
this._replicationInfoBlockedPeers.
|
|
24178
|
+
this._replicationInfoBlockedPeers.add(fromHash);
|
|
22145
24179
|
this.invalidateSharedLogTopicSubscribersCache();
|
|
22146
24180
|
|
|
22147
24181
|
await this.handleSubscriptionChange(
|
|
22148
24182
|
evt.detail.from,
|
|
22149
24183
|
evt.detail.topics,
|
|
22150
24184
|
true,
|
|
24185
|
+
subscriptionEpoch,
|
|
22151
24186
|
);
|
|
22152
24187
|
}
|
|
22153
24188
|
|
|
@@ -22342,7 +24377,7 @@ export class SharedLog<
|
|
|
22342
24377
|
|
|
22343
24378
|
if (ih) {
|
|
22344
24379
|
ih.clear();
|
|
22345
|
-
|
|
24380
|
+
this.runPendingIHaveCallback(ih, entry);
|
|
22346
24381
|
}
|
|
22347
24382
|
|
|
22348
24383
|
this.syncronizer.onEntryAdded(entry);
|
|
@@ -22356,7 +24391,7 @@ export class SharedLog<
|
|
|
22356
24391
|
}
|
|
22357
24392
|
const entry = materializeEntry();
|
|
22358
24393
|
ih.clear();
|
|
22359
|
-
|
|
24394
|
+
this.runPendingIHaveCallback(ih, entry);
|
|
22360
24395
|
this.syncronizer.onEntryAdded(entry);
|
|
22361
24396
|
return;
|
|
22362
24397
|
}
|