@peerbit/shared-log 15.0.0 → 16.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/checked-prune.d.ts.map +1 -1
- package/dist/src/checked-prune.js +12 -0
- package/dist/src/checked-prune.js.map +1 -1
- package/dist/src/index.d.ts +7 -13
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +81 -598
- package/dist/src/index.js.map +1 -1
- package/dist/src/instance-lifecycle.d.ts.map +1 -1
- package/dist/src/instance-lifecycle.js +9 -0
- package/dist/src/instance-lifecycle.js.map +1 -1
- package/dist/src/peer-session.d.ts.map +1 -1
- package/dist/src/peer-session.js +9 -0
- package/dist/src/peer-session.js.map +1 -1
- package/dist/src/pid.d.ts +0 -2
- package/dist/src/pid.d.ts.map +1 -1
- package/dist/src/pid.js +0 -5
- package/dist/src/pid.js.map +1 -1
- package/dist/src/replication-announcement.d.ts +3 -106
- package/dist/src/replication-announcement.d.ts.map +1 -1
- package/dist/src/replication-announcement.js +0 -521
- package/dist/src/replication-announcement.js.map +1 -1
- package/dist/src/replication-info-v2-receive.d.ts +14 -43
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -1
- package/dist/src/replication-info-v2-receive.js +24 -199
- package/dist/src/replication-info-v2-receive.js.map +1 -1
- package/dist/src/replication-info-v2-send.d.ts +0 -1
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +0 -6
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/dist/src/replication.d.ts +0 -1
- package/dist/src/replication.d.ts.map +1 -1
- package/dist/src/replication.js +2 -17
- package/dist/src/replication.js.map +1 -1
- package/dist/src/role.d.ts +0 -4
- package/dist/src/role.d.ts.map +1 -1
- package/dist/src/role.js +0 -9
- package/dist/src/role.js.map +1 -1
- package/dist/src/sync/factory.d.ts +0 -1
- package/dist/src/sync/factory.d.ts.map +1 -1
- package/dist/src/sync/factory.js +20 -36
- package/dist/src/sync/factory.js.map +1 -1
- package/package.json +8 -8
- package/src/checked-prune.ts +12 -0
- package/src/index.ts +104 -817
- package/src/instance-lifecycle.ts +9 -0
- package/src/peer-session.ts +9 -0
- package/src/pid.ts +0 -6
- package/src/replication-announcement.ts +8 -739
- package/src/replication-info-v2-receive.ts +25 -271
- package/src/replication-info-v2-send.ts +0 -7
- package/src/replication.ts +1 -20
- package/src/role.ts +0 -13
- package/src/sync/factory.ts +24 -38
|
@@ -1,650 +1,26 @@
|
|
|
1
|
-
import type { PublicSignKey } from "@peerbit/crypto";
|
|
2
|
-
import { logger as loggerFn } from "@peerbit/logger";
|
|
3
|
-
import type { RPC } from "@peerbit/rpc";
|
|
4
|
-
import {
|
|
5
|
-
AcknowledgeDelivery,
|
|
6
|
-
CONVERGENCE_MESSAGE_PRIORITY,
|
|
7
|
-
DeliveryError,
|
|
8
|
-
} from "@peerbit/stream-interface";
|
|
9
|
-
import { TimeoutError, debounceFixedInterval } from "@peerbit/time";
|
|
10
|
-
import { isNotStartedError } from "./errors.js";
|
|
11
|
-
import type { TransportMessage } from "./message.js";
|
|
12
|
-
import type { ReplicationRangeIndexable } from "./ranges.js";
|
|
13
1
|
import type { ReplicationInfoMutation } from "./replication-info-mutation.js";
|
|
14
|
-
import {
|
|
15
|
-
AddedReplicationSegmentMessage,
|
|
16
|
-
AllReplicatingSegmentsMessage,
|
|
17
|
-
StoppedReplicating,
|
|
18
|
-
} from "./replication.js";
|
|
19
2
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL = 1000;
|
|
23
|
-
const REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL = 1000;
|
|
24
|
-
const REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS = 3;
|
|
25
|
-
// Repair one bounded cohort per mutation generation. The subscriber snapshot
|
|
26
|
-
// is a best-effort cache and can contain thousands of entries, so attempting
|
|
27
|
-
// the whole cache after every role mutation would turn convergence repair into
|
|
28
|
-
// an unbounded burst of separately signed, acknowledged messages. A cursor
|
|
29
|
-
// retained across generations rotates best-effort coverage over later changes.
|
|
30
|
-
const REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION = 8;
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* Replication announcements are best-effort convergence messages. A detached
|
|
34
|
-
* fanout shard can time out even though the shared log itself remains open.
|
|
35
|
-
* Keep retries deliberately limited to concrete TimeoutErrors: abort/close and
|
|
36
|
-
* unexpected programming/data errors must retain their existing semantics.
|
|
37
|
-
*
|
|
38
|
-
* Exact constructor/name checks complement `instanceof` for errors crossing
|
|
39
|
-
* worker or duplicate-package boundaries in browsers.
|
|
40
|
-
*/
|
|
41
|
-
export const isTransientReplicationAnnouncementError = (
|
|
42
|
-
error: unknown,
|
|
43
|
-
seen = new Set<unknown>(),
|
|
44
|
-
): boolean => {
|
|
45
|
-
if (
|
|
46
|
-
error != null &&
|
|
47
|
-
(typeof error === "object" || typeof error === "function")
|
|
48
|
-
) {
|
|
49
|
-
if (seen.has(error)) {
|
|
50
|
-
return false;
|
|
51
|
-
}
|
|
52
|
-
seen.add(error);
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
if (error instanceof TimeoutError) {
|
|
56
|
-
return true;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
const nested = (error as { errors?: unknown })?.errors;
|
|
60
|
-
if (Array.isArray(nested) && nested.length > 0) {
|
|
61
|
-
return nested.every((item) =>
|
|
62
|
-
isTransientReplicationAnnouncementError(item, new Set(seen)),
|
|
63
|
-
);
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
const cause = (error as { cause?: unknown })?.cause;
|
|
67
|
-
if (cause != null && isTransientReplicationAnnouncementError(cause, seen)) {
|
|
68
|
-
return true;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
const constructorName =
|
|
72
|
-
typeof (error as { constructor?: { name?: unknown } })?.constructor
|
|
73
|
-
?.name === "string"
|
|
74
|
-
? (error as { constructor: { name: string } }).constructor.name
|
|
75
|
-
: "";
|
|
76
|
-
const name =
|
|
77
|
-
typeof (error as { name?: unknown })?.name === "string"
|
|
78
|
-
? (error as { name: string }).name
|
|
79
|
-
: "";
|
|
80
|
-
return constructorName === "TimeoutError" || name === "TimeoutError";
|
|
81
|
-
};
|
|
82
|
-
|
|
83
|
-
/**
|
|
84
|
-
* Directed transport-delivery repair is allowed to retry explicit delivery
|
|
85
|
-
* failures in addition to timeouts. A DirectStream ACK confirms receipt of the
|
|
86
|
-
* signed envelope, not successful application by the receiver. Keep this
|
|
87
|
-
* separate from the primary fanout classifier above so replicate() rejection
|
|
88
|
-
* semantics remain unchanged for programming, serialization, and lifecycle
|
|
89
|
-
* errors.
|
|
90
|
-
*/
|
|
91
|
-
const isTransientReplicationAnnouncementRepairError = (
|
|
92
|
-
error: unknown,
|
|
93
|
-
seen = new Set<unknown>(),
|
|
94
|
-
): boolean => {
|
|
95
|
-
if (
|
|
96
|
-
error != null &&
|
|
97
|
-
(typeof error === "object" || typeof error === "function")
|
|
98
|
-
) {
|
|
99
|
-
if (seen.has(error)) {
|
|
100
|
-
return false;
|
|
101
|
-
}
|
|
102
|
-
seen.add(error);
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
if (error instanceof DeliveryError || error instanceof TimeoutError) {
|
|
106
|
-
return true;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const nested = (error as { errors?: unknown })?.errors;
|
|
110
|
-
if (Array.isArray(nested) && nested.length > 0) {
|
|
111
|
-
return nested.every((item) =>
|
|
112
|
-
isTransientReplicationAnnouncementRepairError(item, new Set(seen)),
|
|
113
|
-
);
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
const cause = (error as { cause?: unknown })?.cause;
|
|
117
|
-
if (
|
|
118
|
-
cause != null &&
|
|
119
|
-
isTransientReplicationAnnouncementRepairError(cause, seen)
|
|
120
|
-
) {
|
|
121
|
-
return true;
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
const constructorName =
|
|
125
|
-
typeof (error as { constructor?: { name?: unknown } })?.constructor
|
|
126
|
-
?.name === "string"
|
|
127
|
-
? (error as { constructor: { name: string } }).constructor.name
|
|
128
|
-
: "";
|
|
129
|
-
const name =
|
|
130
|
-
typeof (error as { name?: unknown })?.name === "string"
|
|
131
|
-
? (error as { name: string }).name
|
|
132
|
-
: "";
|
|
133
|
-
return (
|
|
134
|
-
constructorName === "DeliveryError" ||
|
|
135
|
-
name === "DeliveryError" ||
|
|
136
|
-
constructorName === "TimeoutError" ||
|
|
137
|
-
name === "TimeoutError"
|
|
138
|
-
);
|
|
139
|
-
};
|
|
140
|
-
|
|
141
|
-
/**
|
|
142
|
-
* One session per announcement window — rotated at exactly the sites that
|
|
143
|
-
* bumped the legacy retry-generation counter: construction, resetForOpen,
|
|
144
|
-
* cancelCurrentReplicationStateAnnouncementRetry, and the pre-send bump in
|
|
145
|
-
* sendReplicationAnnouncement. Identity comparison replaces every numeric
|
|
146
|
-
* generation compare. Unlike the number, identity cannot alias across open
|
|
147
|
-
* cycles (the legacy reset-to-0 was safe only by the convention that a
|
|
148
|
-
* companion controller check accompanied every compare site).
|
|
149
|
-
*/
|
|
150
|
-
export class AnnouncementWorkerSession {
|
|
151
|
-
readonly createdAt = Date.now(); // diagnostics only
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
type ReplicationAnnouncementRepairTarget = {
|
|
155
|
-
key: PublicSignKey;
|
|
156
|
-
session: AnnouncementWorkerSession;
|
|
157
|
-
attempts: number;
|
|
158
|
-
done: boolean;
|
|
159
|
-
};
|
|
160
|
-
|
|
161
|
-
/**
|
|
162
|
-
* Repair-side adoption of a session. Replaces the legacy compound of the
|
|
163
|
-
* repair generation number, its generation controller, and the repair
|
|
164
|
-
* pending / targets / cohort-selected fields. One binding per adopted
|
|
165
|
-
* session; every rotation aborts the old binding's controller FIRST, then
|
|
166
|
-
* reassigns (abort listeners run synchronously and must observe
|
|
167
|
-
* pre-rotation state, as before). The fair cursor hash and max attempts
|
|
168
|
-
* deliberately stay OUTSIDE the binding: the cursor rotates best-effort
|
|
169
|
-
* coverage ACROSS generations, and max-attempts is a setup-scoped tunable.
|
|
170
|
-
*/
|
|
171
|
-
export type ReplicationAnnouncementRepairBinding = {
|
|
172
|
-
session: AnnouncementWorkerSession;
|
|
173
|
-
controller: AbortController;
|
|
174
|
-
pending: boolean;
|
|
175
|
-
targets: Map<string, ReplicationAnnouncementRepairTarget>;
|
|
176
|
-
cohortSelected: boolean;
|
|
177
|
-
};
|
|
178
|
-
|
|
179
|
-
export type ReplicationAnnouncementRepairWorkerContext = {
|
|
180
|
-
session: AnnouncementWorkerSession;
|
|
181
|
-
lifecycleController: AbortController;
|
|
182
|
-
binding: ReplicationAnnouncementRepairBinding;
|
|
183
|
-
};
|
|
184
|
-
|
|
185
|
-
export type ReplicationAnnouncementDeps<R extends "u32" | "u64"> = {
|
|
186
|
-
isClosed: () => boolean;
|
|
187
|
-
getCloseSignal: () => AbortSignal;
|
|
188
|
-
getMyReplicationSegments: () => Promise<ReplicationRangeIndexable<R>[]>;
|
|
189
|
-
validatePersistedReplicationRangeSnapshot: (
|
|
190
|
-
ranges: readonly { mode: unknown }[],
|
|
191
|
-
) => void;
|
|
192
|
-
getSubscribers: () =>
|
|
193
|
-
| Promise<PublicSignKey[] | undefined>
|
|
194
|
-
| PublicSignKey[]
|
|
195
|
-
| undefined;
|
|
196
|
-
getSelfHash: () => string;
|
|
197
|
-
isBlockedPeer: (hash: string) => boolean;
|
|
198
|
-
getRpc: () => RPC<TransportMessage, TransportMessage>;
|
|
3
|
+
export type ReplicationAnnouncementDeps = {
|
|
199
4
|
captureReplicationOwnershipLifecycle: () => AbortController;
|
|
200
5
|
throwIfReplicationOwnershipLifecycleInactive: (
|
|
201
6
|
controller: AbortController,
|
|
202
7
|
) => void;
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
//
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
// V2 is always current. Legacy publication is enabled only for an explicit
|
|
209
|
-
// compatibility open and retains its exact rejection/retry semantics there.
|
|
8
|
+
// V2 is always current: every committed local mutation feeds the V2
|
|
9
|
+
// sender. The legacy broadcast tail this fed in parallel was deleted in
|
|
10
|
+
// B12 stage 3, and the announcement-session/repair-generation rotation
|
|
11
|
+
// that ordered the legacy retry/repair workers folded away in stage 5
|
|
12
|
+
// once no consumer pinned session identity.
|
|
210
13
|
enqueueReplicationInfoV2: (mutation: ReplicationInfoMutation) => void;
|
|
211
|
-
isLegacyReplicationInfoEnabled: () => boolean;
|
|
212
14
|
};
|
|
213
15
|
|
|
214
|
-
|
|
215
|
-
* The dormant legacy tail still broadcasts the retired wire classes when an
|
|
216
|
-
* explicit compatibility open enabled it (impossible since B12 stage 1; the
|
|
217
|
-
* tail is deleted in stage 3). Materialize the frame from the neutral
|
|
218
|
-
* mutation locally so the tail keeps its exact byte semantics until then.
|
|
219
|
-
*/
|
|
220
|
-
const toLegacyReplicationInfoFrame = (
|
|
221
|
-
mutation: ReplicationInfoMutation,
|
|
222
|
-
):
|
|
223
|
-
| AllReplicatingSegmentsMessage
|
|
224
|
-
| AddedReplicationSegmentMessage
|
|
225
|
-
| StoppedReplicating => {
|
|
226
|
-
if ("full" in mutation) {
|
|
227
|
-
return new AllReplicatingSegmentsMessage({
|
|
228
|
-
segments: mutation.full.segments,
|
|
229
|
-
});
|
|
230
|
-
}
|
|
231
|
-
if ("added" in mutation) {
|
|
232
|
-
return new AddedReplicationSegmentMessage({
|
|
233
|
-
segments: mutation.added.segments,
|
|
234
|
-
});
|
|
235
|
-
}
|
|
236
|
-
return new StoppedReplicating({ segmentIds: mutation.stopped.segmentIds });
|
|
237
|
-
};
|
|
238
|
-
|
|
239
|
-
export class ReplicationAnnouncementCoordinator<R extends "u32" | "u64"> {
|
|
240
|
-
replicationAnnouncementRetryDebounced:
|
|
241
|
-
| ReturnType<typeof debounceFixedInterval>
|
|
242
|
-
| undefined;
|
|
243
|
-
_replicationAnnouncementRetryPending!: boolean;
|
|
244
|
-
_announcementSession!: AnnouncementWorkerSession;
|
|
245
|
-
_replicationAnnouncementRetryController!: AbortController;
|
|
16
|
+
export class ReplicationAnnouncementCoordinator {
|
|
246
17
|
// Publish local ownership announcements in committed mutation order. This
|
|
247
18
|
// prevents an older Added message with a delayed transport completion from
|
|
248
19
|
// overtaking a newer authoritative empty snapshot.
|
|
249
20
|
_replicationAnnouncementSendTails?: WeakMap<AbortController, Promise<void>>;
|
|
250
|
-
replicationAnnouncementRepairDebounced:
|
|
251
|
-
| ReturnType<typeof debounceFixedInterval>
|
|
252
|
-
| undefined;
|
|
253
|
-
_announcementRepairBinding!: ReplicationAnnouncementRepairBinding;
|
|
254
|
-
_replicationAnnouncementRepairFairCursorHash!: string | undefined;
|
|
255
|
-
_replicationAnnouncementRepairMaxAttempts!: number;
|
|
256
|
-
_replicationAnnouncementRepairController!: AbortController;
|
|
257
21
|
|
|
258
|
-
constructor(private readonly deps: ReplicationAnnouncementDeps
|
|
259
|
-
this._replicationAnnouncementRetryPending = false;
|
|
260
|
-
this._announcementSession = new AnnouncementWorkerSession();
|
|
261
|
-
this._replicationAnnouncementRetryController = new AbortController();
|
|
22
|
+
constructor(private readonly deps: ReplicationAnnouncementDeps) {
|
|
262
23
|
this._replicationAnnouncementSendTails = new WeakMap();
|
|
263
|
-
this._announcementRepairBinding = this.createRepairBinding(
|
|
264
|
-
this._announcementSession,
|
|
265
|
-
);
|
|
266
|
-
this._replicationAnnouncementRepairFairCursorHash = undefined;
|
|
267
|
-
this._replicationAnnouncementRepairMaxAttempts =
|
|
268
|
-
REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS;
|
|
269
|
-
this._replicationAnnouncementRepairController = new AbortController();
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
private rotateAnnouncementSession(): AnnouncementWorkerSession {
|
|
273
|
-
return (this._announcementSession = new AnnouncementWorkerSession());
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
private createRepairBinding(
|
|
277
|
-
session: AnnouncementWorkerSession,
|
|
278
|
-
): ReplicationAnnouncementRepairBinding {
|
|
279
|
-
return {
|
|
280
|
-
session,
|
|
281
|
-
controller: new AbortController(),
|
|
282
|
-
pending: false,
|
|
283
|
-
targets: new Map(),
|
|
284
|
-
cohortSelected: false,
|
|
285
|
-
};
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
resetForOpen(): void {
|
|
289
|
-
this._replicationAnnouncementRetryPending = false;
|
|
290
|
-
this.rotateAnnouncementSession();
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
queueCurrentReplicationStateAnnouncementRetry(error: unknown): boolean {
|
|
294
|
-
if (
|
|
295
|
-
!this.deps.isLegacyReplicationInfoEnabled() ||
|
|
296
|
-
this.deps.isClosed() ||
|
|
297
|
-
this.deps.getCloseSignal().aborted ||
|
|
298
|
-
this._replicationAnnouncementRetryController.signal.aborted ||
|
|
299
|
-
!isTransientReplicationAnnouncementError(error)
|
|
300
|
-
) {
|
|
301
|
-
return false;
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
this._replicationAnnouncementRetryPending = true;
|
|
305
|
-
void this.replicationAnnouncementRetryDebounced?.call();
|
|
306
|
-
return true;
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
setupReplicationAnnouncementRetryFunction(
|
|
310
|
-
interval = REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL,
|
|
311
|
-
): void {
|
|
312
|
-
this.replicationAnnouncementRetryDebounced?.close();
|
|
313
|
-
this._replicationAnnouncementRetryController?.abort();
|
|
314
|
-
this._replicationAnnouncementRetryController = new AbortController();
|
|
315
|
-
this.replicationAnnouncementRetryDebounced = debounceFixedInterval(
|
|
316
|
-
() => this.retryCurrentReplicationStateAnnouncement(),
|
|
317
|
-
interval,
|
|
318
|
-
{
|
|
319
|
-
leading: false,
|
|
320
|
-
onError: (error) => {
|
|
321
|
-
if (
|
|
322
|
-
this.deps.isClosed() ||
|
|
323
|
-
this.deps.getCloseSignal().aborted ||
|
|
324
|
-
isNotStartedError(error)
|
|
325
|
-
) {
|
|
326
|
-
return;
|
|
327
|
-
}
|
|
328
|
-
logger.error(error);
|
|
329
|
-
},
|
|
330
|
-
},
|
|
331
|
-
);
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
setupReplicationAnnouncementRepairFunction(
|
|
335
|
-
interval = REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL,
|
|
336
|
-
maxAttempts = REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS,
|
|
337
|
-
): void {
|
|
338
|
-
if (!Number.isSafeInteger(maxAttempts) || maxAttempts <= 0) {
|
|
339
|
-
throw new RangeError(
|
|
340
|
-
"Replication announcement repair attempts must be positive",
|
|
341
|
-
);
|
|
342
|
-
}
|
|
343
|
-
this.replicationAnnouncementRepairDebounced?.close();
|
|
344
|
-
this._replicationAnnouncementRepairController?.abort();
|
|
345
|
-
this._announcementRepairBinding?.controller.abort();
|
|
346
|
-
this._replicationAnnouncementRepairController = new AbortController();
|
|
347
|
-
this._announcementRepairBinding = this.createRepairBinding(
|
|
348
|
-
this._announcementSession,
|
|
349
|
-
);
|
|
350
|
-
this._replicationAnnouncementRepairFairCursorHash = undefined;
|
|
351
|
-
this._replicationAnnouncementRepairMaxAttempts = maxAttempts;
|
|
352
|
-
this.replicationAnnouncementRepairDebounced = debounceFixedInterval(
|
|
353
|
-
() => this.runCurrentReplicationStateAnnouncementRepair(),
|
|
354
|
-
interval,
|
|
355
|
-
{
|
|
356
|
-
leading: false,
|
|
357
|
-
// The wrapper catches worker failures while it still owns the generation
|
|
358
|
-
// context. Keep this boundary visibility-only: it must never mutate a
|
|
359
|
-
// possibly newer generation's pending state.
|
|
360
|
-
onError: (error) => logger.error(error),
|
|
361
|
-
},
|
|
362
|
-
);
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
cancelCurrentReplicationStateAnnouncementRepair(): void {
|
|
366
|
-
this._announcementRepairBinding.pending = false;
|
|
367
|
-
this._replicationAnnouncementRepairController?.abort();
|
|
368
|
-
this._announcementRepairBinding.controller.abort();
|
|
369
|
-
this.replicationAnnouncementRepairDebounced?.close();
|
|
370
|
-
this._announcementRepairBinding.targets.clear();
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
advanceCurrentReplicationStateAnnouncementRepairGeneration(): void {
|
|
374
|
-
const session = this._announcementSession;
|
|
375
|
-
if (session === this._announcementRepairBinding.session) {
|
|
376
|
-
return;
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
// Abort acknowledged sends carrying the old full-state snapshot before the
|
|
380
|
-
// primary announcement for the new mutation waits on transport. Otherwise a
|
|
381
|
-
// stale batch can hold the current state behind DirectStream's seek timeout.
|
|
382
|
-
this._announcementRepairBinding.controller.abort();
|
|
383
|
-
this._announcementRepairBinding = this.createRepairBinding(session);
|
|
384
|
-
}
|
|
385
|
-
|
|
386
|
-
queueCurrentReplicationStateAnnouncementRepair(): void {
|
|
387
|
-
if (
|
|
388
|
-
!this.deps.isLegacyReplicationInfoEnabled() ||
|
|
389
|
-
this.deps.isClosed() ||
|
|
390
|
-
this.deps.getCloseSignal().aborted ||
|
|
391
|
-
this._replicationAnnouncementRepairController.signal.aborted ||
|
|
392
|
-
!this.replicationAnnouncementRepairDebounced
|
|
393
|
-
) {
|
|
394
|
-
return;
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
|
|
398
|
-
this._announcementRepairBinding.pending = true;
|
|
399
|
-
void this.replicationAnnouncementRepairDebounced.call();
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
/**
|
|
403
|
-
* Single validity predicate for announcement-repair workers. "stale":
|
|
404
|
-
* the store closed or a lifecycle controller aborted — exit silently.
|
|
405
|
-
* "superseded": a newer announcement session took over — the worker
|
|
406
|
-
* must requeue so the current session gets serviced. The binding
|
|
407
|
-
* identity comparison stays at the one call site that historically
|
|
408
|
-
* required it; folding it in here is a stage-5 semantic decision, not a
|
|
409
|
-
* consolidation.
|
|
410
|
-
*/
|
|
411
|
-
private announcementRepairWorkerStatus(
|
|
412
|
-
worker: ReplicationAnnouncementRepairWorkerContext,
|
|
413
|
-
): "current" | "stale" | "superseded" {
|
|
414
|
-
if (
|
|
415
|
-
this.deps.isClosed() ||
|
|
416
|
-
this.deps.getCloseSignal().aborted ||
|
|
417
|
-
worker.lifecycleController.signal.aborted ||
|
|
418
|
-
worker.binding.controller.signal.aborted
|
|
419
|
-
) {
|
|
420
|
-
return "stale";
|
|
421
|
-
}
|
|
422
|
-
if (worker.session !== this._announcementSession) {
|
|
423
|
-
return "superseded";
|
|
424
|
-
}
|
|
425
|
-
return "current";
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
async runCurrentReplicationStateAnnouncementRepair(): Promise<void> {
|
|
429
|
-
const session = this._announcementSession;
|
|
430
|
-
const lifecycleController = this._replicationAnnouncementRepairController;
|
|
431
|
-
const binding = this._announcementRepairBinding;
|
|
432
|
-
try {
|
|
433
|
-
await this.repairCurrentReplicationStateAnnouncement({
|
|
434
|
-
session,
|
|
435
|
-
lifecycleController,
|
|
436
|
-
binding,
|
|
437
|
-
});
|
|
438
|
-
} catch (error) {
|
|
439
|
-
if (
|
|
440
|
-
this.announcementRepairWorkerStatus({
|
|
441
|
-
session,
|
|
442
|
-
lifecycleController,
|
|
443
|
-
binding,
|
|
444
|
-
}) !== "current" ||
|
|
445
|
-
binding !== this._announcementRepairBinding
|
|
446
|
-
) {
|
|
447
|
-
return;
|
|
448
|
-
}
|
|
449
|
-
if (isNotStartedError(error as Error)) {
|
|
450
|
-
return;
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
// Only the worker that still owns the current session may conclude
|
|
454
|
-
// that its repair failed. A stale worker must not clear a newer call's
|
|
455
|
-
// pending flag or attribute its error to the new session.
|
|
456
|
-
this._announcementRepairBinding.pending = false;
|
|
457
|
-
logger.error(error);
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
async repairCurrentReplicationStateAnnouncement(
|
|
462
|
-
context?: ReplicationAnnouncementRepairWorkerContext,
|
|
463
|
-
): Promise<void> {
|
|
464
|
-
if (!this.deps.isLegacyReplicationInfoEnabled()) {
|
|
465
|
-
this._announcementRepairBinding.pending = false;
|
|
466
|
-
this._announcementRepairBinding.targets.clear();
|
|
467
|
-
return;
|
|
468
|
-
}
|
|
469
|
-
if (!this._announcementRepairBinding.pending) {
|
|
470
|
-
return;
|
|
471
|
-
}
|
|
472
|
-
const session = context?.session ?? this._announcementSession;
|
|
473
|
-
const lifecycleController =
|
|
474
|
-
context?.lifecycleController ??
|
|
475
|
-
this._replicationAnnouncementRepairController;
|
|
476
|
-
const binding = context?.binding ?? this._announcementRepairBinding;
|
|
477
|
-
const segments = (await this.deps.getMyReplicationSegments()).map((range) =>
|
|
478
|
-
range.toReplicationRange(),
|
|
479
|
-
);
|
|
480
|
-
switch (
|
|
481
|
-
this.announcementRepairWorkerStatus({
|
|
482
|
-
session,
|
|
483
|
-
lifecycleController,
|
|
484
|
-
binding,
|
|
485
|
-
})
|
|
486
|
-
) {
|
|
487
|
-
case "stale":
|
|
488
|
-
return;
|
|
489
|
-
case "superseded":
|
|
490
|
-
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
491
|
-
return;
|
|
492
|
-
}
|
|
493
|
-
this.deps.validatePersistedReplicationRangeSnapshot(segments);
|
|
494
|
-
|
|
495
|
-
const subscribers = (await this.deps.getSubscribers()) ?? [];
|
|
496
|
-
switch (
|
|
497
|
-
this.announcementRepairWorkerStatus({
|
|
498
|
-
session,
|
|
499
|
-
lifecycleController,
|
|
500
|
-
binding,
|
|
501
|
-
})
|
|
502
|
-
) {
|
|
503
|
-
case "stale":
|
|
504
|
-
return;
|
|
505
|
-
case "superseded":
|
|
506
|
-
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
507
|
-
return;
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
const selfHash = this.deps.getSelfHash();
|
|
511
|
-
const currentTargets = new Map<string, PublicSignKey>();
|
|
512
|
-
for (const key of subscribers) {
|
|
513
|
-
const hash = key.hashcode();
|
|
514
|
-
if (
|
|
515
|
-
hash !== selfHash &&
|
|
516
|
-
!this.deps.isBlockedPeer(hash) &&
|
|
517
|
-
!currentTargets.has(hash)
|
|
518
|
-
) {
|
|
519
|
-
currentTargets.set(hash, key);
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
for (const [hash, target] of this._announcementRepairBinding.targets) {
|
|
524
|
-
if (target.session !== session || !currentTargets.has(hash)) {
|
|
525
|
-
this._announcementRepairBinding.targets.delete(hash);
|
|
526
|
-
} else {
|
|
527
|
-
target.key = currentTargets.get(hash)!;
|
|
528
|
-
}
|
|
529
|
-
}
|
|
530
|
-
if (!this._announcementRepairBinding.cohortSelected) {
|
|
531
|
-
const candidates = [...currentTargets.entries()].sort(([left], [right]) =>
|
|
532
|
-
left.localeCompare(right),
|
|
533
|
-
);
|
|
534
|
-
const cursorIndex = this._replicationAnnouncementRepairFairCursorHash
|
|
535
|
-
? candidates.findIndex(
|
|
536
|
-
([hash]) =>
|
|
537
|
-
hash.localeCompare(
|
|
538
|
-
this._replicationAnnouncementRepairFairCursorHash!,
|
|
539
|
-
) > 0,
|
|
540
|
-
)
|
|
541
|
-
: 0;
|
|
542
|
-
const fairStart = cursorIndex < 0 ? 0 : cursorIndex;
|
|
543
|
-
const fairOrder = [
|
|
544
|
-
...candidates.slice(fairStart),
|
|
545
|
-
...candidates.slice(0, fairStart),
|
|
546
|
-
];
|
|
547
|
-
const cohort = fairOrder.slice(
|
|
548
|
-
0,
|
|
549
|
-
REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION,
|
|
550
|
-
);
|
|
551
|
-
for (const [hash, key] of cohort) {
|
|
552
|
-
this._announcementRepairBinding.targets.set(hash, {
|
|
553
|
-
key,
|
|
554
|
-
session,
|
|
555
|
-
attempts: 0,
|
|
556
|
-
done: false,
|
|
557
|
-
});
|
|
558
|
-
}
|
|
559
|
-
if (cohort.length > 0) {
|
|
560
|
-
this._replicationAnnouncementRepairFairCursorHash =
|
|
561
|
-
cohort[cohort.length - 1][0];
|
|
562
|
-
}
|
|
563
|
-
this._announcementRepairBinding.cohortSelected = true;
|
|
564
|
-
}
|
|
565
|
-
|
|
566
|
-
const batch = [...this._announcementRepairBinding.targets.entries()].filter(
|
|
567
|
-
([, target]) => !target.done,
|
|
568
|
-
);
|
|
569
|
-
const snapshot = new AllReplicatingSegmentsMessage({ segments });
|
|
570
|
-
const results = await Promise.allSettled(
|
|
571
|
-
batch.map(([, target]) =>
|
|
572
|
-
this.deps.getRpc().send(snapshot, {
|
|
573
|
-
mode: new AcknowledgeDelivery({
|
|
574
|
-
to: [target.key],
|
|
575
|
-
redundancy: 1,
|
|
576
|
-
}),
|
|
577
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
578
|
-
signal: binding.controller.signal,
|
|
579
|
-
}),
|
|
580
|
-
),
|
|
581
|
-
);
|
|
582
|
-
switch (
|
|
583
|
-
this.announcementRepairWorkerStatus({
|
|
584
|
-
session,
|
|
585
|
-
lifecycleController,
|
|
586
|
-
binding,
|
|
587
|
-
})
|
|
588
|
-
) {
|
|
589
|
-
case "stale":
|
|
590
|
-
return;
|
|
591
|
-
case "superseded":
|
|
592
|
-
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
593
|
-
return;
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
for (const [index, result] of results.entries()) {
|
|
597
|
-
const [hash, attemptedTarget] = batch[index];
|
|
598
|
-
const target = this._announcementRepairBinding.targets.get(hash);
|
|
599
|
-
if (target !== attemptedTarget || target.session !== session) {
|
|
600
|
-
continue;
|
|
601
|
-
}
|
|
602
|
-
if (result.status === "fulfilled") {
|
|
603
|
-
// DirectStream ACKs confirm that the signed transport envelope reached
|
|
604
|
-
// the target. Applying the contained replication state remains a
|
|
605
|
-
// receiver-local, best-effort operation.
|
|
606
|
-
target.done = true;
|
|
607
|
-
continue;
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
target.attempts += 1;
|
|
611
|
-
if (!isTransientReplicationAnnouncementRepairError(result.reason)) {
|
|
612
|
-
target.done = true;
|
|
613
|
-
logger.error(result.reason);
|
|
614
|
-
} else if (
|
|
615
|
-
target.attempts >= this._replicationAnnouncementRepairMaxAttempts
|
|
616
|
-
) {
|
|
617
|
-
target.done = true;
|
|
618
|
-
logger.trace(
|
|
619
|
-
"Acknowledged replication announcement repair exhausted for %s",
|
|
620
|
-
hash,
|
|
621
|
-
);
|
|
622
|
-
}
|
|
623
|
-
}
|
|
624
|
-
|
|
625
|
-
if (session !== this._announcementSession) {
|
|
626
|
-
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
627
|
-
return;
|
|
628
|
-
}
|
|
629
|
-
if (
|
|
630
|
-
[...this._announcementRepairBinding.targets.values()].some(
|
|
631
|
-
(target) => !target.done,
|
|
632
|
-
)
|
|
633
|
-
) {
|
|
634
|
-
void this.replicationAnnouncementRepairDebounced?.call();
|
|
635
|
-
return;
|
|
636
|
-
}
|
|
637
|
-
|
|
638
|
-
this._announcementRepairBinding.pending = false;
|
|
639
|
-
this._announcementRepairBinding.targets.clear();
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
cancelCurrentReplicationStateAnnouncementRetry(): void {
|
|
643
|
-
this.rotateAnnouncementSession();
|
|
644
|
-
this._replicationAnnouncementRetryPending = false;
|
|
645
|
-
this._replicationAnnouncementRetryController?.abort();
|
|
646
|
-
this.replicationAnnouncementRetryDebounced?.close();
|
|
647
|
-
this.cancelCurrentReplicationStateAnnouncementRepair();
|
|
648
24
|
}
|
|
649
25
|
|
|
650
26
|
async sendReplicationAnnouncement(
|
|
@@ -667,38 +43,7 @@ export class ReplicationAnnouncementCoordinator<R extends "u32" | "u64"> {
|
|
|
667
43
|
if (options?.shouldSend && !options.shouldSend()) {
|
|
668
44
|
return;
|
|
669
45
|
}
|
|
670
|
-
// Advance before every post-mutation send, including successful ones. An
|
|
671
|
-
// authoritative retry already in flight may have captured the previous
|
|
672
|
-
// local state; the session mismatch forces one more current snapshot
|
|
673
|
-
// after that stale send settles.
|
|
674
|
-
this.rotateAnnouncementSession();
|
|
675
|
-
this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
|
|
676
46
|
this.deps.enqueueReplicationInfoV2(mutation);
|
|
677
|
-
if (!this.deps.isLegacyReplicationInfoEnabled()) {
|
|
678
|
-
return;
|
|
679
|
-
}
|
|
680
|
-
try {
|
|
681
|
-
await this.deps.getRpc().send(toLegacyReplicationInfoFrame(mutation), {
|
|
682
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
683
|
-
signal: ownershipLifecycleController.signal,
|
|
684
|
-
});
|
|
685
|
-
this.deps.throwIfReplicationOwnershipLifecycleInactive(
|
|
686
|
-
ownershipLifecycleController,
|
|
687
|
-
);
|
|
688
|
-
this.deps.queueCurrentReplicationStateAnnouncementRepair();
|
|
689
|
-
} catch (error) {
|
|
690
|
-
// An old send can reject only after poison or close has installed a new
|
|
691
|
-
// ownership generation. Never enqueue its retry work into that generation.
|
|
692
|
-
this.deps.throwIfReplicationOwnershipLifecycleInactive(
|
|
693
|
-
ownershipLifecycleController,
|
|
694
|
-
);
|
|
695
|
-
// The local replication-index mutation precedes all calls to this
|
|
696
|
-
// wrapper. Preserve the explicit caller's rejection, but independently
|
|
697
|
-
// schedule an authoritative snapshot so peers eventually observe the
|
|
698
|
-
// already-committed local state.
|
|
699
|
-
this.deps.queueCurrentReplicationStateAnnouncementRetry(error);
|
|
700
|
-
throw error;
|
|
701
|
-
}
|
|
702
47
|
});
|
|
703
48
|
// Keep the ordering barrier usable after a caller-observed send rejection.
|
|
704
49
|
tails.set(
|
|
@@ -707,80 +52,4 @@ export class ReplicationAnnouncementCoordinator<R extends "u32" | "u64"> {
|
|
|
707
52
|
);
|
|
708
53
|
return send;
|
|
709
54
|
}
|
|
710
|
-
|
|
711
|
-
async retryCurrentReplicationStateAnnouncement(): Promise<void> {
|
|
712
|
-
if (!this.deps.isLegacyReplicationInfoEnabled()) {
|
|
713
|
-
this._replicationAnnouncementRetryPending = false;
|
|
714
|
-
return;
|
|
715
|
-
}
|
|
716
|
-
const session = this._announcementSession;
|
|
717
|
-
const controller = this._replicationAnnouncementRetryController;
|
|
718
|
-
try {
|
|
719
|
-
const segments = (await this.deps.getMyReplicationSegments()).map(
|
|
720
|
-
(range) => range.toReplicationRange(),
|
|
721
|
-
);
|
|
722
|
-
if (
|
|
723
|
-
this.deps.isClosed() ||
|
|
724
|
-
this.deps.getCloseSignal().aborted ||
|
|
725
|
-
controller.signal.aborted
|
|
726
|
-
) {
|
|
727
|
-
return;
|
|
728
|
-
}
|
|
729
|
-
if (session !== this._announcementSession) {
|
|
730
|
-
void this.replicationAnnouncementRetryDebounced?.call();
|
|
731
|
-
return;
|
|
732
|
-
}
|
|
733
|
-
this.deps.validatePersistedReplicationRangeSnapshot(segments);
|
|
734
|
-
|
|
735
|
-
await this.deps
|
|
736
|
-
.getRpc()
|
|
737
|
-
.send(new AllReplicatingSegmentsMessage({ segments }), {
|
|
738
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
739
|
-
signal: controller.signal,
|
|
740
|
-
});
|
|
741
|
-
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
742
|
-
} catch (error) {
|
|
743
|
-
if (
|
|
744
|
-
this.deps.isClosed() ||
|
|
745
|
-
this.deps.getCloseSignal().aborted ||
|
|
746
|
-
controller.signal.aborted
|
|
747
|
-
) {
|
|
748
|
-
return;
|
|
749
|
-
}
|
|
750
|
-
if (this.queueCurrentReplicationStateAnnouncementRetry(error)) {
|
|
751
|
-
return;
|
|
752
|
-
}
|
|
753
|
-
if (session === this._announcementSession) {
|
|
754
|
-
this._replicationAnnouncementRetryPending = false;
|
|
755
|
-
} else {
|
|
756
|
-
void this.replicationAnnouncementRetryDebounced?.call();
|
|
757
|
-
}
|
|
758
|
-
throw error;
|
|
759
|
-
}
|
|
760
|
-
if (
|
|
761
|
-
this.deps.isClosed() ||
|
|
762
|
-
this.deps.getCloseSignal().aborted ||
|
|
763
|
-
controller.signal.aborted
|
|
764
|
-
) {
|
|
765
|
-
return;
|
|
766
|
-
}
|
|
767
|
-
|
|
768
|
-
// A newer mutation announcement may have started while this snapshot was
|
|
769
|
-
// in flight. In that case keep the repair pending so the newer current
|
|
770
|
-
// state is also announced in full, regardless of whether its incremental
|
|
771
|
-
// send succeeded or failed.
|
|
772
|
-
if (session === this._announcementSession) {
|
|
773
|
-
this._replicationAnnouncementRetryPending = false;
|
|
774
|
-
if (
|
|
775
|
-
!this.deps.isClosed() &&
|
|
776
|
-
!this.deps.getCloseSignal().aborted &&
|
|
777
|
-
!controller.signal.aborted &&
|
|
778
|
-
this.deps.isAdaptiveReplicating()
|
|
779
|
-
) {
|
|
780
|
-
void this.deps.callRebalanceParticipationDebounced();
|
|
781
|
-
}
|
|
782
|
-
} else {
|
|
783
|
-
void this.replicationAnnouncementRetryDebounced?.call();
|
|
784
|
-
}
|
|
785
|
-
}
|
|
786
55
|
}
|