@peerbit/shared-log 13.2.24 → 13.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/checked-prune.d.ts +12 -0
- package/dist/src/checked-prune.d.ts.map +1 -1
- package/dist/src/checked-prune.js +33 -0
- package/dist/src/checked-prune.js.map +1 -1
- package/dist/src/coordinate-persistence.d.ts +211 -0
- package/dist/src/coordinate-persistence.d.ts.map +1 -0
- package/dist/src/coordinate-persistence.js +1133 -0
- package/dist/src/coordinate-persistence.js.map +1 -0
- package/dist/src/errors.d.ts +1 -0
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +19 -0
- package/dist/src/errors.js.map +1 -1
- package/dist/src/index.d.ts +127 -133
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1400 -4269
- package/dist/src/index.js.map +1 -1
- package/dist/src/instance-lifecycle.d.ts +117 -0
- package/dist/src/instance-lifecycle.d.ts.map +1 -0
- package/dist/src/instance-lifecycle.js +254 -0
- package/dist/src/instance-lifecycle.js.map +1 -0
- package/dist/src/join-warmup.d.ts +86 -0
- package/dist/src/join-warmup.d.ts.map +1 -0
- package/dist/src/join-warmup.js +351 -0
- package/dist/src/join-warmup.js.map +1 -0
- package/dist/src/native-write-through-block-store.d.ts +123 -0
- package/dist/src/native-write-through-block-store.d.ts.map +1 -0
- package/dist/src/native-write-through-block-store.js +989 -0
- package/dist/src/native-write-through-block-store.js.map +1 -0
- package/dist/src/peer-session.d.ts +117 -0
- package/dist/src/peer-session.d.ts.map +1 -0
- package/dist/src/peer-session.js +259 -0
- package/dist/src/peer-session.js.map +1 -0
- package/dist/src/replication-announcement.d.ts +116 -0
- package/dist/src/replication-announcement.d.ts.map +1 -0
- package/dist/src/replication-announcement.js +515 -0
- package/dist/src/replication-announcement.js.map +1 -0
- package/dist/src/replicator-liveness.d.ts +59 -0
- package/dist/src/replicator-liveness.d.ts.map +1 -0
- package/dist/src/replicator-liveness.js +304 -0
- package/dist/src/replicator-liveness.js.map +1 -0
- package/dist/src/sync/dispatch-lifecycle.d.ts +48 -0
- package/dist/src/sync/dispatch-lifecycle.d.ts.map +1 -0
- package/dist/src/sync/dispatch-lifecycle.js +141 -0
- package/dist/src/sync/dispatch-lifecycle.js.map +1 -0
- package/dist/src/sync/factory.d.ts +29 -0
- package/dist/src/sync/factory.d.ts.map +1 -0
- package/dist/src/sync/factory.js +87 -0
- package/dist/src/sync/factory.js.map +1 -0
- package/dist/src/sync/pending-sync-store.d.ts +105 -0
- package/dist/src/sync/pending-sync-store.d.ts.map +1 -0
- package/dist/src/sync/pending-sync-store.js +877 -0
- package/dist/src/sync/pending-sync-store.js.map +1 -0
- package/dist/src/sync/rateless-iblt.d.ts +4 -1
- package/dist/src/sync/rateless-iblt.d.ts.map +1 -1
- package/dist/src/sync/rateless-iblt.js +102 -106
- package/dist/src/sync/rateless-iblt.js.map +1 -1
- package/dist/src/sync/simple.d.ts +10 -41
- package/dist/src/sync/simple.d.ts.map +1 -1
- package/dist/src/sync/simple.js +259 -952
- package/dist/src/sync/simple.js.map +1 -1
- package/dist/src/sync/sync-peer-state.d.ts +16 -0
- package/dist/src/sync/sync-peer-state.d.ts.map +1 -0
- package/dist/src/sync/sync-peer-state.js +82 -0
- package/dist/src/sync/sync-peer-state.js.map +1 -0
- package/package.json +12 -12
- package/src/checked-prune.ts +35 -0
- package/src/coordinate-persistence.ts +1796 -0
- package/src/errors.ts +21 -0
- package/src/index.ts +4014 -7858
- package/src/instance-lifecycle.ts +351 -0
- package/src/join-warmup.ts +507 -0
- package/src/native-write-through-block-store.ts +1170 -0
- package/src/peer-session.ts +336 -0
- package/src/replication-announcement.ts +744 -0
- package/src/replicator-liveness.ts +439 -0
- package/src/sync/dispatch-lifecycle.ts +230 -0
- package/src/sync/factory.ts +154 -0
- package/src/sync/pending-sync-store.ts +1085 -0
- package/src/sync/rateless-iblt.ts +122 -132
- package/src/sync/simple.ts +334 -1144
- package/src/sync/sync-peer-state.ts +107 -0
|
@@ -0,0 +1,439 @@
|
|
|
1
|
+
import type { PublicSignKey } from "@peerbit/crypto";
|
|
2
|
+
import { logger as loggerFn } from "@peerbit/logger";
|
|
3
|
+
import { waitForSubscribers } from "@peerbit/pubsub";
|
|
4
|
+
import type { RPC } from "@peerbit/rpc";
|
|
5
|
+
import {
|
|
6
|
+
ACK_CONTROL_PRIORITY,
|
|
7
|
+
AcknowledgeDelivery,
|
|
8
|
+
} from "@peerbit/stream-interface";
|
|
9
|
+
import { isNotStartedError } from "./errors.js";
|
|
10
|
+
import type { TransportMessage } from "./message.js";
|
|
11
|
+
import type { PeerSession } from "./peer-session.js";
|
|
12
|
+
import { ReplicationPingMessage } from "./replication.js";
|
|
13
|
+
|
|
14
|
+
const logger = loggerFn("peerbit:shared-log");
|
|
15
|
+
|
|
16
|
+
// In sparse topologies (browser/relay), peers can learn about replicators via broadcast
|
|
17
|
+
// replication announcements without having a direct connection that emits unsubscribe
|
|
18
|
+
// on abrupt churn. Probe conservatively so a single missed ACK does not evict a
|
|
19
|
+
// healthy replicator, and rely on replication-info refresh to recover membership.
|
|
20
|
+
const REPLICATOR_LIVENESS_SWEEP_INTERVAL_MS = 2_000;
|
|
21
|
+
const REPLICATOR_LIVENESS_IDLE_THRESHOLD_MS = 8_000;
|
|
22
|
+
const REPLICATOR_LIVENESS_PROBE_FAILURES_TO_EVICT = 2;
|
|
23
|
+
|
|
24
|
+
export type ReplicatorLivenessDeps = {
|
|
25
|
+
isClosed: () => boolean;
|
|
26
|
+
getCloseSignal: () => AbortSignal;
|
|
27
|
+
getReplicationLifecycleController: () => AbortController | undefined;
|
|
28
|
+
isReplicationLifecycleActive: (
|
|
29
|
+
controller: AbortController | undefined,
|
|
30
|
+
) => boolean;
|
|
31
|
+
getSelfHash: () => string;
|
|
32
|
+
getUniqueReplicators: () => Set<string>;
|
|
33
|
+
// Stage 3: probes capture the peer's PeerSession (the opaque
|
|
34
|
+
// subscription-epoch token) and check currency through the registry.
|
|
35
|
+
getPeerSession: (peerHash: string) => PeerSession | null;
|
|
36
|
+
isPeerSessionCurrent: (
|
|
37
|
+
peerHash: string,
|
|
38
|
+
session: PeerSession | null,
|
|
39
|
+
) => boolean;
|
|
40
|
+
resolvePublicKeyFromHash: (
|
|
41
|
+
hash: string,
|
|
42
|
+
) => Promise<PublicSignKey | undefined>;
|
|
43
|
+
removeReplicator: (
|
|
44
|
+
key: PublicSignKey | string,
|
|
45
|
+
options?: {
|
|
46
|
+
noEvent?: boolean;
|
|
47
|
+
onRemoved?: (state: { wasReplicator: boolean }) => void;
|
|
48
|
+
replicationLifecycleController?: AbortController;
|
|
49
|
+
shouldRemove?: () => boolean;
|
|
50
|
+
subscriptionEpoch?: PeerSession | null;
|
|
51
|
+
},
|
|
52
|
+
) => Promise<boolean>;
|
|
53
|
+
getRpc: () => RPC<TransportMessage, TransportMessage>;
|
|
54
|
+
getPendingReplicatorLeaveByPeer: () => Set<string>;
|
|
55
|
+
dispatchReplicatorLeave: (publicKey: PublicSignKey) => void;
|
|
56
|
+
isBlockedPeer: (hash: string) => boolean;
|
|
57
|
+
scheduleReplicationInfoRequests: (
|
|
58
|
+
peer: PublicSignKey,
|
|
59
|
+
replicationLifecycleController: AbortController,
|
|
60
|
+
) => void;
|
|
61
|
+
getTopicSubscribers: (
|
|
62
|
+
topic: string,
|
|
63
|
+
) => Promise<PublicSignKey[] | undefined>;
|
|
64
|
+
// Late-bound through the SharedLog instance so tests can override the
|
|
65
|
+
// presence check on the log while probes are in flight.
|
|
66
|
+
confirmReplicatorSubscriberPresence: (peerHash: string) => Promise<boolean>;
|
|
67
|
+
getNode: () => Parameters<typeof waitForSubscribers>[0];
|
|
68
|
+
getWaitForReplicatorTimeout: () => number;
|
|
69
|
+
// Owner-routed dispatch so monitor stubs/spies keep intercepting
|
|
70
|
+
// sweep-driven probes and activity marks.
|
|
71
|
+
probeReplicatorLiveness: (peerHash: string) => Promise<void> | void;
|
|
72
|
+
markReplicatorActivity: (peerHash: string, now?: number) => void;
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
export class ReplicatorLivenessMonitor {
|
|
76
|
+
_replicatorLivenessSweepRunning!: boolean;
|
|
77
|
+
_replicatorLivenessTimer?: ReturnType<typeof setInterval>;
|
|
78
|
+
_replicatorLivenessTargets!: string[];
|
|
79
|
+
_replicatorLivenessTargetsSize!: number;
|
|
80
|
+
_replicatorLivenessCursor!: number;
|
|
81
|
+
_replicatorLivenessFailures!: Map<string, number>;
|
|
82
|
+
_replicatorLastActivityAt!: Map<string, number>;
|
|
83
|
+
|
|
84
|
+
constructor(private readonly deps: ReplicatorLivenessDeps) {
|
|
85
|
+
this._replicatorLivenessSweepRunning = false;
|
|
86
|
+
this._replicatorLivenessTargets = [];
|
|
87
|
+
this._replicatorLivenessTargetsSize = 0;
|
|
88
|
+
this._replicatorLivenessCursor = 0;
|
|
89
|
+
this._replicatorLivenessFailures = new Map();
|
|
90
|
+
this._replicatorLastActivityAt = new Map();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
resetForOpen(): void {
|
|
94
|
+
this._replicatorLivenessSweepRunning = false;
|
|
95
|
+
this._replicatorLivenessTimer = undefined;
|
|
96
|
+
this._replicatorLivenessTargets = [];
|
|
97
|
+
this._replicatorLivenessTargetsSize = 0;
|
|
98
|
+
this._replicatorLivenessCursor = 0;
|
|
99
|
+
this._replicatorLivenessFailures = new Map();
|
|
100
|
+
this._replicatorLastActivityAt = new Map();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
startReplicatorLivenessSweep() {
|
|
104
|
+
if (this._replicatorLivenessTimer) {
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
this._replicatorLivenessTimer = setInterval(() => {
|
|
108
|
+
void this.runReplicatorLivenessSweep();
|
|
109
|
+
}, REPLICATOR_LIVENESS_SWEEP_INTERVAL_MS);
|
|
110
|
+
this._replicatorLivenessTimer.unref?.();
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
stopReplicatorLivenessSweep() {
|
|
114
|
+
if (this._replicatorLivenessTimer) {
|
|
115
|
+
clearInterval(this._replicatorLivenessTimer);
|
|
116
|
+
this._replicatorLivenessTimer = undefined;
|
|
117
|
+
}
|
|
118
|
+
this._replicatorLivenessSweepRunning = false;
|
|
119
|
+
this._replicatorLivenessTargets = [];
|
|
120
|
+
this._replicatorLivenessTargetsSize = 0;
|
|
121
|
+
this._replicatorLivenessCursor = 0;
|
|
122
|
+
this._replicatorLivenessFailures.clear();
|
|
123
|
+
this._replicatorLastActivityAt.clear();
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
private rebuildReplicatorLivenessTargets() {
|
|
127
|
+
const selfHash = this.deps.getSelfHash();
|
|
128
|
+
this._replicatorLivenessTargets = [
|
|
129
|
+
...this.deps.getUniqueReplicators(),
|
|
130
|
+
].filter((hash) => hash !== selfHash);
|
|
131
|
+
this._replicatorLivenessTargetsSize = this.deps.getUniqueReplicators().size;
|
|
132
|
+
if (
|
|
133
|
+
this._replicatorLivenessCursor >= this._replicatorLivenessTargets.length
|
|
134
|
+
) {
|
|
135
|
+
this._replicatorLivenessCursor = 0;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
private getReplicatorLivenessTargets() {
|
|
140
|
+
const selfHash = this.deps.getSelfHash();
|
|
141
|
+
const uniqueReplicators = this.deps.getUniqueReplicators();
|
|
142
|
+
const expected =
|
|
143
|
+
uniqueReplicators.size - (uniqueReplicators.has(selfHash) ? 1 : 0);
|
|
144
|
+
|
|
145
|
+
if (this._replicatorLivenessTargets.length > 0) {
|
|
146
|
+
// Keep the cursor stable, but purge stale hashes (membership can change while
|
|
147
|
+
// the total size stays constant).
|
|
148
|
+
this._replicatorLivenessTargets = this._replicatorLivenessTargets.filter(
|
|
149
|
+
(hash) => hash !== selfHash && uniqueReplicators.has(hash),
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (
|
|
154
|
+
this._replicatorLivenessTargetsSize !== uniqueReplicators.size ||
|
|
155
|
+
this._replicatorLivenessTargets.length !== expected
|
|
156
|
+
) {
|
|
157
|
+
this.rebuildReplicatorLivenessTargets();
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
return this._replicatorLivenessTargets;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
markReplicatorActivity(peerHash: string, now = Date.now()) {
|
|
164
|
+
this._replicatorLastActivityAt.set(peerHash, now);
|
|
165
|
+
// Any recent authenticated activity is positive liveness evidence. Reset the
|
|
166
|
+
// consecutive miss streak immediately, including while an eviction is
|
|
167
|
+
// waiting in the per-peer mutation lane.
|
|
168
|
+
if (Date.now() - now < REPLICATOR_LIVENESS_IDLE_THRESHOLD_MS) {
|
|
169
|
+
this._replicatorLivenessFailures.delete(peerHash);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
private hasRecentReplicatorActivity(peerHash: string, now = Date.now()) {
|
|
174
|
+
const lastActivityAt = this._replicatorLastActivityAt.get(peerHash);
|
|
175
|
+
if (
|
|
176
|
+
lastActivityAt != null &&
|
|
177
|
+
now - lastActivityAt < REPLICATOR_LIVENESS_IDLE_THRESHOLD_MS
|
|
178
|
+
) {
|
|
179
|
+
this._replicatorLivenessFailures.delete(peerHash);
|
|
180
|
+
return true;
|
|
181
|
+
}
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
private async evictReplicatorFromLiveness(
|
|
186
|
+
peerHash: string,
|
|
187
|
+
publicKey: PublicSignKey,
|
|
188
|
+
replicationLifecycleController: AbortController,
|
|
189
|
+
subscriptionEpoch: PeerSession | null,
|
|
190
|
+
observedActivityAt: number | undefined,
|
|
191
|
+
) {
|
|
192
|
+
try {
|
|
193
|
+
await this.deps.removeReplicator(publicKey, {
|
|
194
|
+
noEvent: true,
|
|
195
|
+
replicationLifecycleController,
|
|
196
|
+
shouldRemove: () =>
|
|
197
|
+
this._replicatorLastActivityAt.get(peerHash) === observedActivityAt,
|
|
198
|
+
subscriptionEpoch,
|
|
199
|
+
onRemoved: ({ wasReplicator }) => {
|
|
200
|
+
if (wasReplicator) {
|
|
201
|
+
this.deps.getPendingReplicatorLeaveByPeer().add(peerHash);
|
|
202
|
+
}
|
|
203
|
+
// A newer subscription/lifecycle may have started while the admitted
|
|
204
|
+
// removal was completing. Its reconnect barrier owns all later effects.
|
|
205
|
+
if (
|
|
206
|
+
!this.deps.isReplicationLifecycleActive(
|
|
207
|
+
replicationLifecycleController,
|
|
208
|
+
) ||
|
|
209
|
+
!this.deps.isPeerSessionCurrent(peerHash, subscriptionEpoch)
|
|
210
|
+
) {
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
if (
|
|
214
|
+
this.deps.getPendingReplicatorLeaveByPeer().delete(peerHash)
|
|
215
|
+
) {
|
|
216
|
+
this.deps.dispatchReplicatorLeave(publicKey);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
if (!this.deps.isBlockedPeer(peerHash)) {
|
|
220
|
+
this.deps.scheduleReplicationInfoRequests(
|
|
221
|
+
publicKey,
|
|
222
|
+
replicationLifecycleController,
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
this._replicatorLivenessTargetsSize = -1;
|
|
226
|
+
},
|
|
227
|
+
});
|
|
228
|
+
} catch (error) {
|
|
229
|
+
if (!isNotStartedError(error as Error)) {
|
|
230
|
+
throw error;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
async runReplicatorLivenessSweep() {
|
|
236
|
+
const replicationLifecycleController =
|
|
237
|
+
this.deps.getReplicationLifecycleController();
|
|
238
|
+
if (
|
|
239
|
+
this.deps.isClosed() ||
|
|
240
|
+
this.deps.getCloseSignal().aborted ||
|
|
241
|
+
!this.deps.isReplicationLifecycleActive(replicationLifecycleController)
|
|
242
|
+
) {
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
if (this._replicatorLivenessSweepRunning) {
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const targets = this.getReplicatorLivenessTargets();
|
|
250
|
+
if (targets.length === 0) {
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
this._replicatorLivenessSweepRunning = true;
|
|
255
|
+
try {
|
|
256
|
+
if (this._replicatorLivenessCursor >= targets.length) {
|
|
257
|
+
this._replicatorLivenessCursor = 0;
|
|
258
|
+
}
|
|
259
|
+
const peerHash = targets[this._replicatorLivenessCursor]!;
|
|
260
|
+
this._replicatorLivenessCursor =
|
|
261
|
+
(this._replicatorLivenessCursor + 1) % targets.length;
|
|
262
|
+
await this.deps.probeReplicatorLiveness(peerHash);
|
|
263
|
+
} catch (error) {
|
|
264
|
+
if (!isNotStartedError(error as Error)) {
|
|
265
|
+
logger.error((error as any)?.toString?.() ?? String(error));
|
|
266
|
+
}
|
|
267
|
+
} finally {
|
|
268
|
+
if (
|
|
269
|
+
this.deps.getReplicationLifecycleController() ===
|
|
270
|
+
replicationLifecycleController
|
|
271
|
+
) {
|
|
272
|
+
this._replicatorLivenessSweepRunning = false;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
async probeReplicatorLiveness(peerHash: string) {
|
|
278
|
+
const replicationLifecycleController =
|
|
279
|
+
this.deps.getReplicationLifecycleController();
|
|
280
|
+
if (
|
|
281
|
+
this.deps.isClosed() ||
|
|
282
|
+
this.deps.getCloseSignal().aborted ||
|
|
283
|
+
!replicationLifecycleController ||
|
|
284
|
+
!this.deps.isReplicationLifecycleActive(replicationLifecycleController)
|
|
285
|
+
) {
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
const subscriptionEpoch = this.deps.getPeerSession(peerHash);
|
|
289
|
+
const ownsProbe = () =>
|
|
290
|
+
this.deps.isReplicationLifecycleActive(replicationLifecycleController) &&
|
|
291
|
+
this.deps.isPeerSessionCurrent(peerHash, subscriptionEpoch);
|
|
292
|
+
if (!this.deps.getUniqueReplicators().has(peerHash)) {
|
|
293
|
+
this._replicatorLivenessFailures.delete(peerHash);
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
const observedActivityAt = this._replicatorLastActivityAt.get(peerHash);
|
|
300
|
+
|
|
301
|
+
const publicKey = await this.deps.resolvePublicKeyFromHash(peerHash);
|
|
302
|
+
if (!ownsProbe()) {
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
if (!publicKey) {
|
|
309
|
+
try {
|
|
310
|
+
await this.deps.removeReplicator(peerHash, {
|
|
311
|
+
noEvent: true,
|
|
312
|
+
replicationLifecycleController,
|
|
313
|
+
shouldRemove: () =>
|
|
314
|
+
this._replicatorLastActivityAt.get(peerHash) === observedActivityAt,
|
|
315
|
+
subscriptionEpoch,
|
|
316
|
+
onRemoved: () => {
|
|
317
|
+
if (!ownsProbe()) {
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
this._replicatorLivenessTargetsSize = -1;
|
|
321
|
+
},
|
|
322
|
+
});
|
|
323
|
+
} catch (error) {
|
|
324
|
+
if (!isNotStartedError(error as Error)) {
|
|
325
|
+
throw error;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
try {
|
|
332
|
+
// Explicit ping (ACKed) instead of RequestReplicationInfoMessage to avoid
|
|
333
|
+
// triggering large segment snapshots just to prove liveness.
|
|
334
|
+
await this.deps.getRpc().send(new ReplicationPingMessage(), {
|
|
335
|
+
mode: new AcknowledgeDelivery({ redundancy: 1, to: [publicKey] }),
|
|
336
|
+
priority: ACK_CONTROL_PRIORITY,
|
|
337
|
+
responsePriority: ACK_CONTROL_PRIORITY,
|
|
338
|
+
});
|
|
339
|
+
if (!ownsProbe()) {
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
this.deps.markReplicatorActivity(peerHash);
|
|
343
|
+
this._replicatorLivenessFailures.delete(peerHash);
|
|
344
|
+
return;
|
|
345
|
+
} catch (error) {
|
|
346
|
+
if (isNotStartedError(error as Error)) {
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
if (!ownsProbe()) {
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// Relay-backed prod paths can keep a peer subscribed/reachable even if an
|
|
358
|
+
// ACKed liveness ping gets delayed or dropped under load. Treat observed
|
|
359
|
+
// topic presence as a positive liveness signal before evicting the peer.
|
|
360
|
+
if (await this.deps.confirmReplicatorSubscriberPresence(peerHash)) {
|
|
361
|
+
if (!ownsProbe()) {
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
this.deps.markReplicatorActivity(peerHash);
|
|
365
|
+
this._replicatorLivenessFailures.delete(peerHash);
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
if (!ownsProbe()) {
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
if (this.hasRecentReplicatorActivity(peerHash)) {
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
const failures = (this._replicatorLivenessFailures.get(peerHash) ?? 0) + 1;
|
|
376
|
+
this._replicatorLivenessFailures.set(peerHash, failures);
|
|
377
|
+
this.deps.scheduleReplicationInfoRequests(
|
|
378
|
+
publicKey,
|
|
379
|
+
replicationLifecycleController,
|
|
380
|
+
);
|
|
381
|
+
|
|
382
|
+
if (failures < REPLICATOR_LIVENESS_PROBE_FAILURES_TO_EVICT) {
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
if (!ownsProbe() || !this.deps.getUniqueReplicators().has(peerHash)) {
|
|
386
|
+
this._replicatorLivenessFailures.delete(peerHash);
|
|
387
|
+
return;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
await this.evictReplicatorFromLiveness(
|
|
391
|
+
peerHash,
|
|
392
|
+
publicKey,
|
|
393
|
+
replicationLifecycleController,
|
|
394
|
+
subscriptionEpoch,
|
|
395
|
+
observedActivityAt,
|
|
396
|
+
);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
async confirmReplicatorSubscriberPresence(peerHash: string) {
|
|
400
|
+
try {
|
|
401
|
+
const subscribers = await this.deps.getTopicSubscribers(
|
|
402
|
+
this.deps.getRpc().topic,
|
|
403
|
+
);
|
|
404
|
+
if (
|
|
405
|
+
subscribers?.some((subscriber) => subscriber.hashcode() === peerHash)
|
|
406
|
+
) {
|
|
407
|
+
return true;
|
|
408
|
+
}
|
|
409
|
+
} catch (error) {
|
|
410
|
+
if (isNotStartedError(error as Error)) {
|
|
411
|
+
return false;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
try {
|
|
416
|
+
await waitForSubscribers(
|
|
417
|
+
this.deps.getNode(),
|
|
418
|
+
peerHash,
|
|
419
|
+
this.deps.getRpc().topic,
|
|
420
|
+
{
|
|
421
|
+
signal: this.deps.getCloseSignal(),
|
|
422
|
+
timeout: Math.max(
|
|
423
|
+
1_000,
|
|
424
|
+
Math.min(
|
|
425
|
+
5_000,
|
|
426
|
+
Math.floor(this.deps.getWaitForReplicatorTimeout() / 4),
|
|
427
|
+
),
|
|
428
|
+
),
|
|
429
|
+
},
|
|
430
|
+
);
|
|
431
|
+
return true;
|
|
432
|
+
} catch (error) {
|
|
433
|
+
if (isNotStartedError(error as Error)) {
|
|
434
|
+
return false;
|
|
435
|
+
}
|
|
436
|
+
return false;
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
// Shared dispatch-lifecycle registry for the sync module (stage-4 sync
|
|
2
|
+
// unification). The simple and rateless synchronizers used to carry
|
|
3
|
+
// near-verbatim copies of the capture/abort/finish/dispose/isActive
|
|
4
|
+
// lifecycle machinery; this registry hosts the shared mechanics and keeps
|
|
5
|
+
// the two deliberate behavioral deltas as injected strategies:
|
|
6
|
+
//
|
|
7
|
+
// - target activity: the simple synchronizer compares the target's dispatch
|
|
8
|
+
// EPOCH identity, the rateless synchronizer checks SET MEMBERSHIP of the
|
|
9
|
+
// target lifecycle in the active-target registry. The strategy is always
|
|
10
|
+
// injected (`isTargetCurrent`), never defaulted.
|
|
11
|
+
// - retention: disposal is deferred while `hasRetainedWork` is true — a
|
|
12
|
+
// retained-work counter for the simple synchronizer, a per-target
|
|
13
|
+
// retainedByProcess/responseLeases scan for the rateless one.
|
|
14
|
+
//
|
|
15
|
+
// Lifecycle objects are constructed by the callers (they carry
|
|
16
|
+
// caller-specific fields such as epochs, batches and retained-work
|
|
17
|
+
// counters); the registry owns the per-target active sets, the listener
|
|
18
|
+
// add/remove pairing and the dispose gating.
|
|
19
|
+
|
|
20
|
+
export interface DispatchTargetLifecycleBase<LC> {
|
|
21
|
+
lifecycle: LC;
|
|
22
|
+
target: string;
|
|
23
|
+
controller: AbortController;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface DispatchLifecycleBase<TL> {
|
|
27
|
+
ownershipLifecycleController: AbortController;
|
|
28
|
+
callerSignal?: AbortSignal;
|
|
29
|
+
controller: AbortController;
|
|
30
|
+
targets: Map<string, TL>;
|
|
31
|
+
onOwnerOrCallerAbort: () => void;
|
|
32
|
+
dispatchFinished: boolean;
|
|
33
|
+
disposed: boolean;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export type DispatchLifecycleRegistryDeps<
|
|
37
|
+
LC extends DispatchLifecycleBase<TL>,
|
|
38
|
+
TL extends DispatchTargetLifecycleBase<LC>,
|
|
39
|
+
> = {
|
|
40
|
+
isClosed: () => boolean;
|
|
41
|
+
currentOwnershipLifecycleController: () => AbortController;
|
|
42
|
+
// Retention hook: disposal is deferred while this is true.
|
|
43
|
+
hasRetainedWork: (lifecycle: LC) => boolean;
|
|
44
|
+
// Target-activity strategy (epoch identity vs set membership).
|
|
45
|
+
isTargetCurrent: (targetLifecycle: TL) => boolean;
|
|
46
|
+
// Side effects on target abort (the simple synchronizer removes the
|
|
47
|
+
// target's pending response batches here).
|
|
48
|
+
onTargetAborted?: (targetLifecycle: TL) => void;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
export class DispatchLifecycleRegistry<
|
|
52
|
+
LC extends DispatchLifecycleBase<TL>,
|
|
53
|
+
TL extends DispatchTargetLifecycleBase<LC>,
|
|
54
|
+
> {
|
|
55
|
+
readonly activeTargets: Map<string, Set<TL>> = new Map();
|
|
56
|
+
|
|
57
|
+
constructor(private readonly deps: DispatchLifecycleRegistryDeps<LC, TL>) {}
|
|
58
|
+
|
|
59
|
+
// Registers the caller-constructed lifecycle: creates the per-target
|
|
60
|
+
// lifecycles (the factory may skip a target or abort it right after
|
|
61
|
+
// registration via `onTargetRegistered`), pairs the owner/caller abort
|
|
62
|
+
// listeners and performs the initial staleness abort exactly like both
|
|
63
|
+
// former capture implementations.
|
|
64
|
+
register(
|
|
65
|
+
lifecycle: LC,
|
|
66
|
+
targets: string[],
|
|
67
|
+
createTargetLifecycle: (lifecycle: LC, target: string) => TL | undefined,
|
|
68
|
+
onTargetRegistered?: (targetLifecycle: TL) => void,
|
|
69
|
+
): void {
|
|
70
|
+
for (const target of [...new Set(targets)]) {
|
|
71
|
+
const targetLifecycle = createTargetLifecycle(lifecycle, target);
|
|
72
|
+
if (!targetLifecycle) {
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
lifecycle.targets.set(target, targetLifecycle);
|
|
76
|
+
let activeForTarget = this.activeTargets.get(target);
|
|
77
|
+
if (!activeForTarget) {
|
|
78
|
+
activeForTarget = new Set();
|
|
79
|
+
this.activeTargets.set(target, activeForTarget);
|
|
80
|
+
}
|
|
81
|
+
activeForTarget.add(targetLifecycle);
|
|
82
|
+
onTargetRegistered?.(targetLifecycle);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
lifecycle.ownershipLifecycleController.signal.addEventListener(
|
|
86
|
+
"abort",
|
|
87
|
+
lifecycle.onOwnerOrCallerAbort,
|
|
88
|
+
{ once: true },
|
|
89
|
+
);
|
|
90
|
+
if (
|
|
91
|
+
lifecycle.callerSignal &&
|
|
92
|
+
lifecycle.callerSignal !== lifecycle.ownershipLifecycleController.signal
|
|
93
|
+
) {
|
|
94
|
+
lifecycle.callerSignal.addEventListener(
|
|
95
|
+
"abort",
|
|
96
|
+
lifecycle.onOwnerOrCallerAbort,
|
|
97
|
+
{ once: true },
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
if (
|
|
101
|
+
this.deps.isClosed() ||
|
|
102
|
+
lifecycle.ownershipLifecycleController !==
|
|
103
|
+
this.deps.currentOwnershipLifecycleController() ||
|
|
104
|
+
lifecycle.ownershipLifecycleController.signal.aborted ||
|
|
105
|
+
lifecycle.callerSignal?.aborted
|
|
106
|
+
) {
|
|
107
|
+
lifecycle.onOwnerOrCallerAbort();
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
abortTarget(targetLifecycle: TL, reason?: unknown): void {
|
|
112
|
+
if (!targetLifecycle.controller.signal.aborted) {
|
|
113
|
+
targetLifecycle.controller.abort(reason);
|
|
114
|
+
}
|
|
115
|
+
this.deps.onTargetAborted?.(targetLifecycle);
|
|
116
|
+
this.maybeDispose(targetLifecycle.lifecycle);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
abortLifecycle(lifecycle: LC, reason?: unknown): void {
|
|
120
|
+
if (!lifecycle.controller.signal.aborted) {
|
|
121
|
+
lifecycle.controller.abort(reason);
|
|
122
|
+
}
|
|
123
|
+
for (const targetLifecycle of lifecycle.targets.values()) {
|
|
124
|
+
this.abortTarget(targetLifecycle, reason);
|
|
125
|
+
}
|
|
126
|
+
this.maybeDispose(lifecycle);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
finish(lifecycle: LC): void {
|
|
130
|
+
lifecycle.dispatchFinished = true;
|
|
131
|
+
this.maybeDispose(lifecycle);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
maybeDispose(lifecycle: LC): void {
|
|
135
|
+
if (
|
|
136
|
+
lifecycle.disposed ||
|
|
137
|
+
!lifecycle.dispatchFinished ||
|
|
138
|
+
this.deps.hasRetainedWork(lifecycle)
|
|
139
|
+
) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
lifecycle.disposed = true;
|
|
143
|
+
lifecycle.ownershipLifecycleController.signal.removeEventListener(
|
|
144
|
+
"abort",
|
|
145
|
+
lifecycle.onOwnerOrCallerAbort,
|
|
146
|
+
);
|
|
147
|
+
if (
|
|
148
|
+
lifecycle.callerSignal &&
|
|
149
|
+
lifecycle.callerSignal !== lifecycle.ownershipLifecycleController.signal
|
|
150
|
+
) {
|
|
151
|
+
lifecycle.callerSignal.removeEventListener(
|
|
152
|
+
"abort",
|
|
153
|
+
lifecycle.onOwnerOrCallerAbort,
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
for (const targetLifecycle of lifecycle.targets.values()) {
|
|
157
|
+
const activeForTarget = this.activeTargets.get(targetLifecycle.target);
|
|
158
|
+
activeForTarget?.delete(targetLifecycle);
|
|
159
|
+
if (activeForTarget?.size === 0) {
|
|
160
|
+
this.activeTargets.delete(targetLifecycle.target);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
isLifecycleActive(lifecycle: LC, target?: string): boolean {
|
|
166
|
+
if (
|
|
167
|
+
this.deps.isClosed() ||
|
|
168
|
+
lifecycle.disposed ||
|
|
169
|
+
lifecycle.ownershipLifecycleController !==
|
|
170
|
+
this.deps.currentOwnershipLifecycleController() ||
|
|
171
|
+
lifecycle.ownershipLifecycleController.signal.aborted ||
|
|
172
|
+
lifecycle.callerSignal?.aborted ||
|
|
173
|
+
lifecycle.controller.signal.aborted
|
|
174
|
+
) {
|
|
175
|
+
return false;
|
|
176
|
+
}
|
|
177
|
+
if (target === undefined) {
|
|
178
|
+
return true;
|
|
179
|
+
}
|
|
180
|
+
const targetLifecycle = lifecycle.targets.get(target);
|
|
181
|
+
return (
|
|
182
|
+
targetLifecycle !== undefined &&
|
|
183
|
+
!targetLifecycle.controller.signal.aborted &&
|
|
184
|
+
this.deps.isTargetCurrent(targetLifecycle)
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
getSignal(lifecycle: LC, target: string): AbortSignal {
|
|
189
|
+
return (
|
|
190
|
+
lifecycle.targets.get(target)?.controller.signal ??
|
|
191
|
+
lifecycle.controller.signal
|
|
192
|
+
);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// The shared ownership-generation identity check: a captured ownership
|
|
197
|
+
// controller is current while the synchronizer is open, the controller is
|
|
198
|
+
// still THE controller (identity, not just non-aborted) and it has not been
|
|
199
|
+
// aborted.
|
|
200
|
+
export const isOwnershipGenerationActive = (properties: {
|
|
201
|
+
closed: boolean;
|
|
202
|
+
ownershipLifecycleController: AbortController;
|
|
203
|
+
currentOwnershipLifecycleController: AbortController;
|
|
204
|
+
}): boolean =>
|
|
205
|
+
!properties.closed &&
|
|
206
|
+
properties.ownershipLifecycleController ===
|
|
207
|
+
properties.currentOwnershipLifecycleController &&
|
|
208
|
+
!properties.ownershipLifecycleController.signal.aborted;
|
|
209
|
+
|
|
210
|
+
// The shared tracked-session identity check (the pattern the host-side
|
|
211
|
+
// instance lifecycle drained the index.ts fences into): a session is active
|
|
212
|
+
// while nothing cancelled or aborted it, its ownership generation is still
|
|
213
|
+
// current, and the session registry still maps its id to THIS session
|
|
214
|
+
// object.
|
|
215
|
+
export const isTrackedSessionActive = <TSession>(properties: {
|
|
216
|
+
closed: boolean;
|
|
217
|
+
cancelled: boolean;
|
|
218
|
+
sessionController: AbortController;
|
|
219
|
+
ownershipLifecycleController: AbortController;
|
|
220
|
+
currentOwnershipLifecycleController: AbortController;
|
|
221
|
+
session: TSession;
|
|
222
|
+
registered: TSession | undefined;
|
|
223
|
+
}): boolean =>
|
|
224
|
+
!properties.closed &&
|
|
225
|
+
!properties.cancelled &&
|
|
226
|
+
!properties.sessionController.signal.aborted &&
|
|
227
|
+
!properties.ownershipLifecycleController.signal.aborted &&
|
|
228
|
+
properties.ownershipLifecycleController ===
|
|
229
|
+
properties.currentOwnershipLifecycleController &&
|
|
230
|
+
properties.registered === properties.session;
|