@peerbit/shared-log 14.0.0 → 16.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/index.d.ts +11 -15
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +84 -596
- package/dist/src/index.js.map +1 -1
- package/dist/src/replication-announcement.d.ts +6 -110
- package/dist/src/replication-announcement.d.ts.map +1 -1
- package/dist/src/replication-announcement.js +2 -504
- package/dist/src/replication-announcement.js.map +1 -1
- package/dist/src/replication-info-mutation.d.ts +34 -0
- package/dist/src/replication-info-mutation.d.ts.map +1 -0
- package/dist/src/replication-info-mutation.js +2 -0
- package/dist/src/replication-info-mutation.js.map +1 -0
- package/dist/src/replication-info-v2-receive.d.ts +10 -41
- package/dist/src/replication-info-v2-receive.d.ts.map +1 -1
- package/dist/src/replication-info-v2-receive.js +11 -192
- package/dist/src/replication-info-v2-receive.js.map +1 -1
- package/dist/src/replication-info-v2-send.d.ts +5 -6
- package/dist/src/replication-info-v2-send.d.ts.map +1 -1
- package/dist/src/replication-info-v2-send.js +9 -14
- package/dist/src/replication-info-v2-send.js.map +1 -1
- package/dist/src/replication.d.ts +0 -1
- package/dist/src/replication.d.ts.map +1 -1
- package/dist/src/replication.js +2 -17
- package/dist/src/replication.js.map +1 -1
- package/dist/src/sync/factory.d.ts +0 -1
- package/dist/src/sync/factory.d.ts.map +1 -1
- package/dist/src/sync/factory.js +20 -36
- package/dist/src/sync/factory.js.map +1 -1
- package/package.json +10 -10
- package/src/index.ts +115 -816
- package/src/replication-announcement.ts +12 -722
- package/src/replication-info-mutation.ts +34 -0
- package/src/replication-info-v2-receive.ts +17 -263
- package/src/replication-info-v2-send.ts +10 -23
- package/src/replication.ts +1 -20
- package/src/sync/factory.ts +24 -38
|
@@ -1,119 +1,15 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
3
|
-
import { debounceFixedInterval } from "@peerbit/time";
|
|
4
|
-
import type { TransportMessage } from "./message.js";
|
|
5
|
-
import type { ReplicationRangeIndexable } from "./ranges.js";
|
|
6
|
-
import { AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, StoppedReplicating } from "./replication.js";
|
|
7
|
-
/**
|
|
8
|
-
* Replication announcements are best-effort convergence messages. A detached
|
|
9
|
-
* fanout shard can time out even though the shared log itself remains open.
|
|
10
|
-
* Keep retries deliberately limited to concrete TimeoutErrors: abort/close and
|
|
11
|
-
* unexpected programming/data errors must retain their existing semantics.
|
|
12
|
-
*
|
|
13
|
-
* Exact constructor/name checks complement `instanceof` for errors crossing
|
|
14
|
-
* worker or duplicate-package boundaries in browsers.
|
|
15
|
-
*/
|
|
16
|
-
export declare const isTransientReplicationAnnouncementError: (error: unknown, seen?: Set<unknown>) => boolean;
|
|
17
|
-
/**
|
|
18
|
-
* One session per announcement window — rotated at exactly the sites that
|
|
19
|
-
* bumped the legacy retry-generation counter: construction, resetForOpen,
|
|
20
|
-
* cancelCurrentReplicationStateAnnouncementRetry, and the pre-send bump in
|
|
21
|
-
* sendReplicationAnnouncement. Identity comparison replaces every numeric
|
|
22
|
-
* generation compare. Unlike the number, identity cannot alias across open
|
|
23
|
-
* cycles (the legacy reset-to-0 was safe only by the convention that a
|
|
24
|
-
* companion controller check accompanied every compare site).
|
|
25
|
-
*/
|
|
26
|
-
export declare class AnnouncementWorkerSession {
|
|
27
|
-
readonly createdAt: number;
|
|
28
|
-
}
|
|
29
|
-
type ReplicationAnnouncementRepairTarget = {
|
|
30
|
-
key: PublicSignKey;
|
|
31
|
-
session: AnnouncementWorkerSession;
|
|
32
|
-
attempts: number;
|
|
33
|
-
done: boolean;
|
|
34
|
-
};
|
|
35
|
-
/**
|
|
36
|
-
* Repair-side adoption of a session. Replaces the legacy compound of the
|
|
37
|
-
* repair generation number, its generation controller, and the repair
|
|
38
|
-
* pending / targets / cohort-selected fields. One binding per adopted
|
|
39
|
-
* session; every rotation aborts the old binding's controller FIRST, then
|
|
40
|
-
* reassigns (abort listeners run synchronously and must observe
|
|
41
|
-
* pre-rotation state, as before). The fair cursor hash and max attempts
|
|
42
|
-
* deliberately stay OUTSIDE the binding: the cursor rotates best-effort
|
|
43
|
-
* coverage ACROSS generations, and max-attempts is a setup-scoped tunable.
|
|
44
|
-
*/
|
|
45
|
-
export type ReplicationAnnouncementRepairBinding = {
|
|
46
|
-
session: AnnouncementWorkerSession;
|
|
47
|
-
controller: AbortController;
|
|
48
|
-
pending: boolean;
|
|
49
|
-
targets: Map<string, ReplicationAnnouncementRepairTarget>;
|
|
50
|
-
cohortSelected: boolean;
|
|
51
|
-
};
|
|
52
|
-
export type ReplicationAnnouncementRepairWorkerContext = {
|
|
53
|
-
session: AnnouncementWorkerSession;
|
|
54
|
-
lifecycleController: AbortController;
|
|
55
|
-
binding: ReplicationAnnouncementRepairBinding;
|
|
56
|
-
};
|
|
57
|
-
export type ReplicationAnnouncementDeps<R extends "u32" | "u64"> = {
|
|
58
|
-
isClosed: () => boolean;
|
|
59
|
-
getCloseSignal: () => AbortSignal;
|
|
60
|
-
getMyReplicationSegments: () => Promise<ReplicationRangeIndexable<R>[]>;
|
|
61
|
-
validatePersistedReplicationRangeSnapshot: (ranges: readonly {
|
|
62
|
-
mode: unknown;
|
|
63
|
-
}[]) => void;
|
|
64
|
-
getSubscribers: () => Promise<PublicSignKey[] | undefined> | PublicSignKey[] | undefined;
|
|
65
|
-
getSelfHash: () => string;
|
|
66
|
-
isBlockedPeer: (hash: string) => boolean;
|
|
67
|
-
getRpc: () => RPC<TransportMessage, TransportMessage>;
|
|
1
|
+
import type { ReplicationInfoMutation } from "./replication-info-mutation.js";
|
|
2
|
+
export type ReplicationAnnouncementDeps = {
|
|
68
3
|
captureReplicationOwnershipLifecycle: () => AbortController;
|
|
69
4
|
throwIfReplicationOwnershipLifecycleInactive: (controller: AbortController) => void;
|
|
70
|
-
|
|
71
|
-
callRebalanceParticipationDebounced: () => unknown;
|
|
72
|
-
queueCurrentReplicationStateAnnouncementRepair: () => void;
|
|
73
|
-
queueCurrentReplicationStateAnnouncementRetry: (error: unknown) => boolean;
|
|
74
|
-
enqueueReplicationInfoV2: (message: LegacyReplicationInfoMessage) => void;
|
|
75
|
-
isLegacyReplicationInfoEnabled: () => boolean;
|
|
5
|
+
enqueueReplicationInfoV2: (mutation: ReplicationInfoMutation) => void;
|
|
76
6
|
};
|
|
77
|
-
|
|
78
|
-
export declare class ReplicationAnnouncementCoordinator<R extends "u32" | "u64"> {
|
|
7
|
+
export declare class ReplicationAnnouncementCoordinator {
|
|
79
8
|
private readonly deps;
|
|
80
|
-
replicationAnnouncementRetryDebounced: ReturnType<typeof debounceFixedInterval> | undefined;
|
|
81
|
-
_replicationAnnouncementRetryPending: boolean;
|
|
82
|
-
_announcementSession: AnnouncementWorkerSession;
|
|
83
|
-
_replicationAnnouncementRetryController: AbortController;
|
|
84
9
|
_replicationAnnouncementSendTails?: WeakMap<AbortController, Promise<void>>;
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
_replicationAnnouncementRepairFairCursorHash: string | undefined;
|
|
88
|
-
_replicationAnnouncementRepairMaxAttempts: number;
|
|
89
|
-
_replicationAnnouncementRepairController: AbortController;
|
|
90
|
-
constructor(deps: ReplicationAnnouncementDeps<R>);
|
|
91
|
-
private rotateAnnouncementSession;
|
|
92
|
-
private createRepairBinding;
|
|
93
|
-
resetForOpen(): void;
|
|
94
|
-
queueCurrentReplicationStateAnnouncementRetry(error: unknown): boolean;
|
|
95
|
-
setupReplicationAnnouncementRetryFunction(interval?: number): void;
|
|
96
|
-
setupReplicationAnnouncementRepairFunction(interval?: number, maxAttempts?: number): void;
|
|
97
|
-
cancelCurrentReplicationStateAnnouncementRepair(): void;
|
|
98
|
-
advanceCurrentReplicationStateAnnouncementRepairGeneration(): void;
|
|
99
|
-
queueCurrentReplicationStateAnnouncementRepair(): void;
|
|
100
|
-
/**
|
|
101
|
-
* Single validity predicate for announcement-repair workers. "stale":
|
|
102
|
-
* the store closed or a lifecycle controller aborted — exit silently.
|
|
103
|
-
* "superseded": a newer announcement session took over — the worker
|
|
104
|
-
* must requeue so the current session gets serviced. The binding
|
|
105
|
-
* identity comparison stays at the one call site that historically
|
|
106
|
-
* required it; folding it in here is a stage-5 semantic decision, not a
|
|
107
|
-
* consolidation.
|
|
108
|
-
*/
|
|
109
|
-
private announcementRepairWorkerStatus;
|
|
110
|
-
runCurrentReplicationStateAnnouncementRepair(): Promise<void>;
|
|
111
|
-
repairCurrentReplicationStateAnnouncement(context?: ReplicationAnnouncementRepairWorkerContext): Promise<void>;
|
|
112
|
-
cancelCurrentReplicationStateAnnouncementRetry(): void;
|
|
113
|
-
sendReplicationAnnouncement(message: LegacyReplicationInfoMessage, ownershipLifecycleController?: AbortController, options?: {
|
|
10
|
+
constructor(deps: ReplicationAnnouncementDeps);
|
|
11
|
+
sendReplicationAnnouncement(mutation: ReplicationInfoMutation, ownershipLifecycleController?: AbortController, options?: {
|
|
114
12
|
shouldSend?: () => boolean;
|
|
115
13
|
}): Promise<void>;
|
|
116
|
-
retryCurrentReplicationStateAnnouncement(): Promise<void>;
|
|
117
14
|
}
|
|
118
|
-
export {};
|
|
119
15
|
//# sourceMappingURL=replication-announcement.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replication-announcement.d.ts","sourceRoot":"","sources":["../../src/replication-announcement.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"replication-announcement.d.ts","sourceRoot":"","sources":["../../src/replication-announcement.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,gCAAgC,CAAC;AAE9E,MAAM,MAAM,2BAA2B,GAAG;IACzC,oCAAoC,EAAE,MAAM,eAAe,CAAC;IAC5D,4CAA4C,EAAE,CAC7C,UAAU,EAAE,eAAe,KACvB,IAAI,CAAC;IAMV,wBAAwB,EAAE,CAAC,QAAQ,EAAE,uBAAuB,KAAK,IAAI,CAAC;CACtE,CAAC;AAEF,qBAAa,kCAAkC;IAMlC,OAAO,CAAC,QAAQ,CAAC,IAAI;IAFjC,iCAAiC,CAAC,EAAE,OAAO,CAAC,eAAe,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;gBAE/C,IAAI,EAAE,2BAA2B;IAIxD,2BAA2B,CAChC,QAAQ,EAAE,uBAAuB,EACjC,4BAA4B,kBAAmD,EAC/E,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,OAAO,CAAA;KAAE,GACtC,OAAO,CAAC,IAAI,CAAC;CAyBhB"}
|
|
@@ -1,422 +1,14 @@
|
|
|
1
|
-
import { logger as loggerFn } from "@peerbit/logger";
|
|
2
|
-
import { AcknowledgeDelivery, CONVERGENCE_MESSAGE_PRIORITY, DeliveryError, } from "@peerbit/stream-interface";
|
|
3
|
-
import { TimeoutError, debounceFixedInterval } from "@peerbit/time";
|
|
4
|
-
import { isNotStartedError } from "./errors.js";
|
|
5
|
-
import { AddedReplicationSegmentMessage, AllReplicatingSegmentsMessage, StoppedReplicating, } from "./replication.js";
|
|
6
|
-
const logger = loggerFn("peerbit:shared-log");
|
|
7
|
-
const REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL = 1000;
|
|
8
|
-
const REPLICATION_ANNOUNCEMENT_REPAIR_INTERVAL = 1000;
|
|
9
|
-
const REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS = 3;
|
|
10
|
-
// Repair one bounded cohort per mutation generation. The subscriber snapshot
|
|
11
|
-
// is a best-effort cache and can contain thousands of entries, so attempting
|
|
12
|
-
// the whole cache after every role mutation would turn convergence repair into
|
|
13
|
-
// an unbounded burst of separately signed, acknowledged messages. A cursor
|
|
14
|
-
// retained across generations rotates best-effort coverage over later changes.
|
|
15
|
-
const REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION = 8;
|
|
16
|
-
/**
|
|
17
|
-
* Replication announcements are best-effort convergence messages. A detached
|
|
18
|
-
* fanout shard can time out even though the shared log itself remains open.
|
|
19
|
-
* Keep retries deliberately limited to concrete TimeoutErrors: abort/close and
|
|
20
|
-
* unexpected programming/data errors must retain their existing semantics.
|
|
21
|
-
*
|
|
22
|
-
* Exact constructor/name checks complement `instanceof` for errors crossing
|
|
23
|
-
* worker or duplicate-package boundaries in browsers.
|
|
24
|
-
*/
|
|
25
|
-
export const isTransientReplicationAnnouncementError = (error, seen = new Set()) => {
|
|
26
|
-
if (error != null &&
|
|
27
|
-
(typeof error === "object" || typeof error === "function")) {
|
|
28
|
-
if (seen.has(error)) {
|
|
29
|
-
return false;
|
|
30
|
-
}
|
|
31
|
-
seen.add(error);
|
|
32
|
-
}
|
|
33
|
-
if (error instanceof TimeoutError) {
|
|
34
|
-
return true;
|
|
35
|
-
}
|
|
36
|
-
const nested = error?.errors;
|
|
37
|
-
if (Array.isArray(nested) && nested.length > 0) {
|
|
38
|
-
return nested.every((item) => isTransientReplicationAnnouncementError(item, new Set(seen)));
|
|
39
|
-
}
|
|
40
|
-
const cause = error?.cause;
|
|
41
|
-
if (cause != null && isTransientReplicationAnnouncementError(cause, seen)) {
|
|
42
|
-
return true;
|
|
43
|
-
}
|
|
44
|
-
const constructorName = typeof error?.constructor
|
|
45
|
-
?.name === "string"
|
|
46
|
-
? error.constructor.name
|
|
47
|
-
: "";
|
|
48
|
-
const name = typeof error?.name === "string"
|
|
49
|
-
? error.name
|
|
50
|
-
: "";
|
|
51
|
-
return constructorName === "TimeoutError" || name === "TimeoutError";
|
|
52
|
-
};
|
|
53
|
-
/**
|
|
54
|
-
* Directed transport-delivery repair is allowed to retry explicit delivery
|
|
55
|
-
* failures in addition to timeouts. A DirectStream ACK confirms receipt of the
|
|
56
|
-
* signed envelope, not successful application by the receiver. Keep this
|
|
57
|
-
* separate from the primary fanout classifier above so replicate() rejection
|
|
58
|
-
* semantics remain unchanged for programming, serialization, and lifecycle
|
|
59
|
-
* errors.
|
|
60
|
-
*/
|
|
61
|
-
const isTransientReplicationAnnouncementRepairError = (error, seen = new Set()) => {
|
|
62
|
-
if (error != null &&
|
|
63
|
-
(typeof error === "object" || typeof error === "function")) {
|
|
64
|
-
if (seen.has(error)) {
|
|
65
|
-
return false;
|
|
66
|
-
}
|
|
67
|
-
seen.add(error);
|
|
68
|
-
}
|
|
69
|
-
if (error instanceof DeliveryError || error instanceof TimeoutError) {
|
|
70
|
-
return true;
|
|
71
|
-
}
|
|
72
|
-
const nested = error?.errors;
|
|
73
|
-
if (Array.isArray(nested) && nested.length > 0) {
|
|
74
|
-
return nested.every((item) => isTransientReplicationAnnouncementRepairError(item, new Set(seen)));
|
|
75
|
-
}
|
|
76
|
-
const cause = error?.cause;
|
|
77
|
-
if (cause != null &&
|
|
78
|
-
isTransientReplicationAnnouncementRepairError(cause, seen)) {
|
|
79
|
-
return true;
|
|
80
|
-
}
|
|
81
|
-
const constructorName = typeof error?.constructor
|
|
82
|
-
?.name === "string"
|
|
83
|
-
? error.constructor.name
|
|
84
|
-
: "";
|
|
85
|
-
const name = typeof error?.name === "string"
|
|
86
|
-
? error.name
|
|
87
|
-
: "";
|
|
88
|
-
return (constructorName === "DeliveryError" ||
|
|
89
|
-
name === "DeliveryError" ||
|
|
90
|
-
constructorName === "TimeoutError" ||
|
|
91
|
-
name === "TimeoutError");
|
|
92
|
-
};
|
|
93
|
-
/**
|
|
94
|
-
* One session per announcement window — rotated at exactly the sites that
|
|
95
|
-
* bumped the legacy retry-generation counter: construction, resetForOpen,
|
|
96
|
-
* cancelCurrentReplicationStateAnnouncementRetry, and the pre-send bump in
|
|
97
|
-
* sendReplicationAnnouncement. Identity comparison replaces every numeric
|
|
98
|
-
* generation compare. Unlike the number, identity cannot alias across open
|
|
99
|
-
* cycles (the legacy reset-to-0 was safe only by the convention that a
|
|
100
|
-
* companion controller check accompanied every compare site).
|
|
101
|
-
*/
|
|
102
|
-
export class AnnouncementWorkerSession {
|
|
103
|
-
createdAt = Date.now(); // diagnostics only
|
|
104
|
-
}
|
|
105
1
|
export class ReplicationAnnouncementCoordinator {
|
|
106
2
|
deps;
|
|
107
|
-
replicationAnnouncementRetryDebounced;
|
|
108
|
-
_replicationAnnouncementRetryPending;
|
|
109
|
-
_announcementSession;
|
|
110
|
-
_replicationAnnouncementRetryController;
|
|
111
3
|
// Publish local ownership announcements in committed mutation order. This
|
|
112
4
|
// prevents an older Added message with a delayed transport completion from
|
|
113
5
|
// overtaking a newer authoritative empty snapshot.
|
|
114
6
|
_replicationAnnouncementSendTails;
|
|
115
|
-
replicationAnnouncementRepairDebounced;
|
|
116
|
-
_announcementRepairBinding;
|
|
117
|
-
_replicationAnnouncementRepairFairCursorHash;
|
|
118
|
-
_replicationAnnouncementRepairMaxAttempts;
|
|
119
|
-
_replicationAnnouncementRepairController;
|
|
120
7
|
constructor(deps) {
|
|
121
8
|
this.deps = deps;
|
|
122
|
-
this._replicationAnnouncementRetryPending = false;
|
|
123
|
-
this._announcementSession = new AnnouncementWorkerSession();
|
|
124
|
-
this._replicationAnnouncementRetryController = new AbortController();
|
|
125
9
|
this._replicationAnnouncementSendTails = new WeakMap();
|
|
126
|
-
this._announcementRepairBinding = this.createRepairBinding(this._announcementSession);
|
|
127
|
-
this._replicationAnnouncementRepairFairCursorHash = undefined;
|
|
128
|
-
this._replicationAnnouncementRepairMaxAttempts =
|
|
129
|
-
REPLICATION_ANNOUNCEMENT_REPAIR_MAX_ATTEMPTS;
|
|
130
|
-
this._replicationAnnouncementRepairController = new AbortController();
|
|
131
|
-
}
|
|
132
|
-
rotateAnnouncementSession() {
|
|
133
|
-
return (this._announcementSession = new AnnouncementWorkerSession());
|
|
134
|
-
}
|
|
135
|
-
createRepairBinding(session) {
|
|
136
|
-
return {
|
|
137
|
-
session,
|
|
138
|
-
controller: new AbortController(),
|
|
139
|
-
pending: false,
|
|
140
|
-
targets: new Map(),
|
|
141
|
-
cohortSelected: false,
|
|
142
|
-
};
|
|
143
|
-
}
|
|
144
|
-
resetForOpen() {
|
|
145
|
-
this._replicationAnnouncementRetryPending = false;
|
|
146
|
-
this.rotateAnnouncementSession();
|
|
147
|
-
}
|
|
148
|
-
queueCurrentReplicationStateAnnouncementRetry(error) {
|
|
149
|
-
if (!this.deps.isLegacyReplicationInfoEnabled() ||
|
|
150
|
-
this.deps.isClosed() ||
|
|
151
|
-
this.deps.getCloseSignal().aborted ||
|
|
152
|
-
this._replicationAnnouncementRetryController.signal.aborted ||
|
|
153
|
-
!isTransientReplicationAnnouncementError(error)) {
|
|
154
|
-
return false;
|
|
155
|
-
}
|
|
156
|
-
this._replicationAnnouncementRetryPending = true;
|
|
157
|
-
void this.replicationAnnouncementRetryDebounced?.call();
|
|
158
|
-
return true;
|
|
159
|
-
}
|
|
160
|
-
setupReplicationAnnouncementRetryFunction(interval = REPLICATION_ANNOUNCEMENT_RETRY_INTERVAL) {
|
|
161
|
-
this.replicationAnnouncementRetryDebounced?.close();
|
|
162
|
-
this._replicationAnnouncementRetryController?.abort();
|
|
163
|
-
this._replicationAnnouncementRetryController = new AbortController();
|
|
164
|
-
this.replicationAnnouncementRetryDebounced = debounceFixedInterval(() => this.retryCurrentReplicationStateAnnouncement(), interval, {
|
|
165
|
-
leading: false,
|
|
166
|
-
onError: (error) => {
|
|
167
|
-
if (this.deps.isClosed() ||
|
|
168
|
-
this.deps.getCloseSignal().aborted ||
|
|
169
|
-
isNotStartedError(error)) {
|
|
170
|
-
return;
|
|
171
|
-
}
|
|
172
|
-
logger.error(error);
|
|
173
|
-
},
|
|
174
|
-
});
|
|
175
10
|
}
|
|
176
|
-
|
|
177
|
-
if (!Number.isSafeInteger(maxAttempts) || maxAttempts <= 0) {
|
|
178
|
-
throw new RangeError("Replication announcement repair attempts must be positive");
|
|
179
|
-
}
|
|
180
|
-
this.replicationAnnouncementRepairDebounced?.close();
|
|
181
|
-
this._replicationAnnouncementRepairController?.abort();
|
|
182
|
-
this._announcementRepairBinding?.controller.abort();
|
|
183
|
-
this._replicationAnnouncementRepairController = new AbortController();
|
|
184
|
-
this._announcementRepairBinding = this.createRepairBinding(this._announcementSession);
|
|
185
|
-
this._replicationAnnouncementRepairFairCursorHash = undefined;
|
|
186
|
-
this._replicationAnnouncementRepairMaxAttempts = maxAttempts;
|
|
187
|
-
this.replicationAnnouncementRepairDebounced = debounceFixedInterval(() => this.runCurrentReplicationStateAnnouncementRepair(), interval, {
|
|
188
|
-
leading: false,
|
|
189
|
-
// The wrapper catches worker failures while it still owns the generation
|
|
190
|
-
// context. Keep this boundary visibility-only: it must never mutate a
|
|
191
|
-
// possibly newer generation's pending state.
|
|
192
|
-
onError: (error) => logger.error(error),
|
|
193
|
-
});
|
|
194
|
-
}
|
|
195
|
-
cancelCurrentReplicationStateAnnouncementRepair() {
|
|
196
|
-
this._announcementRepairBinding.pending = false;
|
|
197
|
-
this._replicationAnnouncementRepairController?.abort();
|
|
198
|
-
this._announcementRepairBinding.controller.abort();
|
|
199
|
-
this.replicationAnnouncementRepairDebounced?.close();
|
|
200
|
-
this._announcementRepairBinding.targets.clear();
|
|
201
|
-
}
|
|
202
|
-
advanceCurrentReplicationStateAnnouncementRepairGeneration() {
|
|
203
|
-
const session = this._announcementSession;
|
|
204
|
-
if (session === this._announcementRepairBinding.session) {
|
|
205
|
-
return;
|
|
206
|
-
}
|
|
207
|
-
// Abort acknowledged sends carrying the old full-state snapshot before the
|
|
208
|
-
// primary announcement for the new mutation waits on transport. Otherwise a
|
|
209
|
-
// stale batch can hold the current state behind DirectStream's seek timeout.
|
|
210
|
-
this._announcementRepairBinding.controller.abort();
|
|
211
|
-
this._announcementRepairBinding = this.createRepairBinding(session);
|
|
212
|
-
}
|
|
213
|
-
queueCurrentReplicationStateAnnouncementRepair() {
|
|
214
|
-
if (!this.deps.isLegacyReplicationInfoEnabled() ||
|
|
215
|
-
this.deps.isClosed() ||
|
|
216
|
-
this.deps.getCloseSignal().aborted ||
|
|
217
|
-
this._replicationAnnouncementRepairController.signal.aborted ||
|
|
218
|
-
!this.replicationAnnouncementRepairDebounced) {
|
|
219
|
-
return;
|
|
220
|
-
}
|
|
221
|
-
this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
|
|
222
|
-
this._announcementRepairBinding.pending = true;
|
|
223
|
-
void this.replicationAnnouncementRepairDebounced.call();
|
|
224
|
-
}
|
|
225
|
-
/**
|
|
226
|
-
* Single validity predicate for announcement-repair workers. "stale":
|
|
227
|
-
* the store closed or a lifecycle controller aborted — exit silently.
|
|
228
|
-
* "superseded": a newer announcement session took over — the worker
|
|
229
|
-
* must requeue so the current session gets serviced. The binding
|
|
230
|
-
* identity comparison stays at the one call site that historically
|
|
231
|
-
* required it; folding it in here is a stage-5 semantic decision, not a
|
|
232
|
-
* consolidation.
|
|
233
|
-
*/
|
|
234
|
-
announcementRepairWorkerStatus(worker) {
|
|
235
|
-
if (this.deps.isClosed() ||
|
|
236
|
-
this.deps.getCloseSignal().aborted ||
|
|
237
|
-
worker.lifecycleController.signal.aborted ||
|
|
238
|
-
worker.binding.controller.signal.aborted) {
|
|
239
|
-
return "stale";
|
|
240
|
-
}
|
|
241
|
-
if (worker.session !== this._announcementSession) {
|
|
242
|
-
return "superseded";
|
|
243
|
-
}
|
|
244
|
-
return "current";
|
|
245
|
-
}
|
|
246
|
-
async runCurrentReplicationStateAnnouncementRepair() {
|
|
247
|
-
const session = this._announcementSession;
|
|
248
|
-
const lifecycleController = this._replicationAnnouncementRepairController;
|
|
249
|
-
const binding = this._announcementRepairBinding;
|
|
250
|
-
try {
|
|
251
|
-
await this.repairCurrentReplicationStateAnnouncement({
|
|
252
|
-
session,
|
|
253
|
-
lifecycleController,
|
|
254
|
-
binding,
|
|
255
|
-
});
|
|
256
|
-
}
|
|
257
|
-
catch (error) {
|
|
258
|
-
if (this.announcementRepairWorkerStatus({
|
|
259
|
-
session,
|
|
260
|
-
lifecycleController,
|
|
261
|
-
binding,
|
|
262
|
-
}) !== "current" ||
|
|
263
|
-
binding !== this._announcementRepairBinding) {
|
|
264
|
-
return;
|
|
265
|
-
}
|
|
266
|
-
if (isNotStartedError(error)) {
|
|
267
|
-
return;
|
|
268
|
-
}
|
|
269
|
-
// Only the worker that still owns the current session may conclude
|
|
270
|
-
// that its repair failed. A stale worker must not clear a newer call's
|
|
271
|
-
// pending flag or attribute its error to the new session.
|
|
272
|
-
this._announcementRepairBinding.pending = false;
|
|
273
|
-
logger.error(error);
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
async repairCurrentReplicationStateAnnouncement(context) {
|
|
277
|
-
if (!this.deps.isLegacyReplicationInfoEnabled()) {
|
|
278
|
-
this._announcementRepairBinding.pending = false;
|
|
279
|
-
this._announcementRepairBinding.targets.clear();
|
|
280
|
-
return;
|
|
281
|
-
}
|
|
282
|
-
if (!this._announcementRepairBinding.pending) {
|
|
283
|
-
return;
|
|
284
|
-
}
|
|
285
|
-
const session = context?.session ?? this._announcementSession;
|
|
286
|
-
const lifecycleController = context?.lifecycleController ??
|
|
287
|
-
this._replicationAnnouncementRepairController;
|
|
288
|
-
const binding = context?.binding ?? this._announcementRepairBinding;
|
|
289
|
-
const segments = (await this.deps.getMyReplicationSegments()).map((range) => range.toReplicationRange());
|
|
290
|
-
switch (this.announcementRepairWorkerStatus({
|
|
291
|
-
session,
|
|
292
|
-
lifecycleController,
|
|
293
|
-
binding,
|
|
294
|
-
})) {
|
|
295
|
-
case "stale":
|
|
296
|
-
return;
|
|
297
|
-
case "superseded":
|
|
298
|
-
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
299
|
-
return;
|
|
300
|
-
}
|
|
301
|
-
this.deps.validatePersistedReplicationRangeSnapshot(segments);
|
|
302
|
-
const subscribers = (await this.deps.getSubscribers()) ?? [];
|
|
303
|
-
switch (this.announcementRepairWorkerStatus({
|
|
304
|
-
session,
|
|
305
|
-
lifecycleController,
|
|
306
|
-
binding,
|
|
307
|
-
})) {
|
|
308
|
-
case "stale":
|
|
309
|
-
return;
|
|
310
|
-
case "superseded":
|
|
311
|
-
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
312
|
-
return;
|
|
313
|
-
}
|
|
314
|
-
const selfHash = this.deps.getSelfHash();
|
|
315
|
-
const currentTargets = new Map();
|
|
316
|
-
for (const key of subscribers) {
|
|
317
|
-
const hash = key.hashcode();
|
|
318
|
-
if (hash !== selfHash &&
|
|
319
|
-
!this.deps.isBlockedPeer(hash) &&
|
|
320
|
-
!currentTargets.has(hash)) {
|
|
321
|
-
currentTargets.set(hash, key);
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
for (const [hash, target] of this._announcementRepairBinding.targets) {
|
|
325
|
-
if (target.session !== session || !currentTargets.has(hash)) {
|
|
326
|
-
this._announcementRepairBinding.targets.delete(hash);
|
|
327
|
-
}
|
|
328
|
-
else {
|
|
329
|
-
target.key = currentTargets.get(hash);
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
if (!this._announcementRepairBinding.cohortSelected) {
|
|
333
|
-
const candidates = [...currentTargets.entries()].sort(([left], [right]) => left.localeCompare(right));
|
|
334
|
-
const cursorIndex = this._replicationAnnouncementRepairFairCursorHash
|
|
335
|
-
? candidates.findIndex(([hash]) => hash.localeCompare(this._replicationAnnouncementRepairFairCursorHash) > 0)
|
|
336
|
-
: 0;
|
|
337
|
-
const fairStart = cursorIndex < 0 ? 0 : cursorIndex;
|
|
338
|
-
const fairOrder = [
|
|
339
|
-
...candidates.slice(fairStart),
|
|
340
|
-
...candidates.slice(0, fairStart),
|
|
341
|
-
];
|
|
342
|
-
const cohort = fairOrder.slice(0, REPLICATION_ANNOUNCEMENT_REPAIR_TARGETS_PER_GENERATION);
|
|
343
|
-
for (const [hash, key] of cohort) {
|
|
344
|
-
this._announcementRepairBinding.targets.set(hash, {
|
|
345
|
-
key,
|
|
346
|
-
session,
|
|
347
|
-
attempts: 0,
|
|
348
|
-
done: false,
|
|
349
|
-
});
|
|
350
|
-
}
|
|
351
|
-
if (cohort.length > 0) {
|
|
352
|
-
this._replicationAnnouncementRepairFairCursorHash =
|
|
353
|
-
cohort[cohort.length - 1][0];
|
|
354
|
-
}
|
|
355
|
-
this._announcementRepairBinding.cohortSelected = true;
|
|
356
|
-
}
|
|
357
|
-
const batch = [...this._announcementRepairBinding.targets.entries()].filter(([, target]) => !target.done);
|
|
358
|
-
const snapshot = new AllReplicatingSegmentsMessage({ segments });
|
|
359
|
-
const results = await Promise.allSettled(batch.map(([, target]) => this.deps.getRpc().send(snapshot, {
|
|
360
|
-
mode: new AcknowledgeDelivery({
|
|
361
|
-
to: [target.key],
|
|
362
|
-
redundancy: 1,
|
|
363
|
-
}),
|
|
364
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
365
|
-
signal: binding.controller.signal,
|
|
366
|
-
})));
|
|
367
|
-
switch (this.announcementRepairWorkerStatus({
|
|
368
|
-
session,
|
|
369
|
-
lifecycleController,
|
|
370
|
-
binding,
|
|
371
|
-
})) {
|
|
372
|
-
case "stale":
|
|
373
|
-
return;
|
|
374
|
-
case "superseded":
|
|
375
|
-
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
376
|
-
return;
|
|
377
|
-
}
|
|
378
|
-
for (const [index, result] of results.entries()) {
|
|
379
|
-
const [hash, attemptedTarget] = batch[index];
|
|
380
|
-
const target = this._announcementRepairBinding.targets.get(hash);
|
|
381
|
-
if (target !== attemptedTarget || target.session !== session) {
|
|
382
|
-
continue;
|
|
383
|
-
}
|
|
384
|
-
if (result.status === "fulfilled") {
|
|
385
|
-
// DirectStream ACKs confirm that the signed transport envelope reached
|
|
386
|
-
// the target. Applying the contained replication state remains a
|
|
387
|
-
// receiver-local, best-effort operation.
|
|
388
|
-
target.done = true;
|
|
389
|
-
continue;
|
|
390
|
-
}
|
|
391
|
-
target.attempts += 1;
|
|
392
|
-
if (!isTransientReplicationAnnouncementRepairError(result.reason)) {
|
|
393
|
-
target.done = true;
|
|
394
|
-
logger.error(result.reason);
|
|
395
|
-
}
|
|
396
|
-
else if (target.attempts >= this._replicationAnnouncementRepairMaxAttempts) {
|
|
397
|
-
target.done = true;
|
|
398
|
-
logger.trace("Acknowledged replication announcement repair exhausted for %s", hash);
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
if (session !== this._announcementSession) {
|
|
402
|
-
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
403
|
-
return;
|
|
404
|
-
}
|
|
405
|
-
if ([...this._announcementRepairBinding.targets.values()].some((target) => !target.done)) {
|
|
406
|
-
void this.replicationAnnouncementRepairDebounced?.call();
|
|
407
|
-
return;
|
|
408
|
-
}
|
|
409
|
-
this._announcementRepairBinding.pending = false;
|
|
410
|
-
this._announcementRepairBinding.targets.clear();
|
|
411
|
-
}
|
|
412
|
-
cancelCurrentReplicationStateAnnouncementRetry() {
|
|
413
|
-
this.rotateAnnouncementSession();
|
|
414
|
-
this._replicationAnnouncementRetryPending = false;
|
|
415
|
-
this._replicationAnnouncementRetryController?.abort();
|
|
416
|
-
this.replicationAnnouncementRetryDebounced?.close();
|
|
417
|
-
this.cancelCurrentReplicationStateAnnouncementRepair();
|
|
418
|
-
}
|
|
419
|
-
async sendReplicationAnnouncement(message, ownershipLifecycleController = this.deps.captureReplicationOwnershipLifecycle(), options) {
|
|
11
|
+
async sendReplicationAnnouncement(mutation, ownershipLifecycleController = this.deps.captureReplicationOwnershipLifecycle(), options) {
|
|
420
12
|
const tails = (this._replicationAnnouncementSendTails ??= new WeakMap());
|
|
421
13
|
const previous = tails.get(ownershipLifecycleController) ?? Promise.resolve();
|
|
422
14
|
const send = previous
|
|
@@ -426,105 +18,11 @@ export class ReplicationAnnouncementCoordinator {
|
|
|
426
18
|
if (options?.shouldSend && !options.shouldSend()) {
|
|
427
19
|
return;
|
|
428
20
|
}
|
|
429
|
-
|
|
430
|
-
// authoritative retry already in flight may have captured the previous
|
|
431
|
-
// local state; the session mismatch forces one more current snapshot
|
|
432
|
-
// after that stale send settles.
|
|
433
|
-
this.rotateAnnouncementSession();
|
|
434
|
-
this.advanceCurrentReplicationStateAnnouncementRepairGeneration();
|
|
435
|
-
this.deps.enqueueReplicationInfoV2(message);
|
|
436
|
-
if (!this.deps.isLegacyReplicationInfoEnabled()) {
|
|
437
|
-
return;
|
|
438
|
-
}
|
|
439
|
-
try {
|
|
440
|
-
await this.deps.getRpc().send(message, {
|
|
441
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
442
|
-
signal: ownershipLifecycleController.signal,
|
|
443
|
-
});
|
|
444
|
-
this.deps.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
445
|
-
this.deps.queueCurrentReplicationStateAnnouncementRepair();
|
|
446
|
-
}
|
|
447
|
-
catch (error) {
|
|
448
|
-
// An old send can reject only after poison or close has installed a new
|
|
449
|
-
// ownership generation. Never enqueue its retry work into that generation.
|
|
450
|
-
this.deps.throwIfReplicationOwnershipLifecycleInactive(ownershipLifecycleController);
|
|
451
|
-
// The local replication-index mutation precedes all calls to this
|
|
452
|
-
// wrapper. Preserve the explicit caller's rejection, but independently
|
|
453
|
-
// schedule an authoritative snapshot so peers eventually observe the
|
|
454
|
-
// already-committed local state.
|
|
455
|
-
this.deps.queueCurrentReplicationStateAnnouncementRetry(error);
|
|
456
|
-
throw error;
|
|
457
|
-
}
|
|
21
|
+
this.deps.enqueueReplicationInfoV2(mutation);
|
|
458
22
|
});
|
|
459
23
|
// Keep the ordering barrier usable after a caller-observed send rejection.
|
|
460
24
|
tails.set(ownershipLifecycleController, send.catch(() => { }));
|
|
461
25
|
return send;
|
|
462
26
|
}
|
|
463
|
-
async retryCurrentReplicationStateAnnouncement() {
|
|
464
|
-
if (!this.deps.isLegacyReplicationInfoEnabled()) {
|
|
465
|
-
this._replicationAnnouncementRetryPending = false;
|
|
466
|
-
return;
|
|
467
|
-
}
|
|
468
|
-
const session = this._announcementSession;
|
|
469
|
-
const controller = this._replicationAnnouncementRetryController;
|
|
470
|
-
try {
|
|
471
|
-
const segments = (await this.deps.getMyReplicationSegments()).map((range) => range.toReplicationRange());
|
|
472
|
-
if (this.deps.isClosed() ||
|
|
473
|
-
this.deps.getCloseSignal().aborted ||
|
|
474
|
-
controller.signal.aborted) {
|
|
475
|
-
return;
|
|
476
|
-
}
|
|
477
|
-
if (session !== this._announcementSession) {
|
|
478
|
-
void this.replicationAnnouncementRetryDebounced?.call();
|
|
479
|
-
return;
|
|
480
|
-
}
|
|
481
|
-
this.deps.validatePersistedReplicationRangeSnapshot(segments);
|
|
482
|
-
await this.deps
|
|
483
|
-
.getRpc()
|
|
484
|
-
.send(new AllReplicatingSegmentsMessage({ segments }), {
|
|
485
|
-
priority: CONVERGENCE_MESSAGE_PRIORITY,
|
|
486
|
-
signal: controller.signal,
|
|
487
|
-
});
|
|
488
|
-
this.queueCurrentReplicationStateAnnouncementRepair();
|
|
489
|
-
}
|
|
490
|
-
catch (error) {
|
|
491
|
-
if (this.deps.isClosed() ||
|
|
492
|
-
this.deps.getCloseSignal().aborted ||
|
|
493
|
-
controller.signal.aborted) {
|
|
494
|
-
return;
|
|
495
|
-
}
|
|
496
|
-
if (this.queueCurrentReplicationStateAnnouncementRetry(error)) {
|
|
497
|
-
return;
|
|
498
|
-
}
|
|
499
|
-
if (session === this._announcementSession) {
|
|
500
|
-
this._replicationAnnouncementRetryPending = false;
|
|
501
|
-
}
|
|
502
|
-
else {
|
|
503
|
-
void this.replicationAnnouncementRetryDebounced?.call();
|
|
504
|
-
}
|
|
505
|
-
throw error;
|
|
506
|
-
}
|
|
507
|
-
if (this.deps.isClosed() ||
|
|
508
|
-
this.deps.getCloseSignal().aborted ||
|
|
509
|
-
controller.signal.aborted) {
|
|
510
|
-
return;
|
|
511
|
-
}
|
|
512
|
-
// A newer mutation announcement may have started while this snapshot was
|
|
513
|
-
// in flight. In that case keep the repair pending so the newer current
|
|
514
|
-
// state is also announced in full, regardless of whether its incremental
|
|
515
|
-
// send succeeded or failed.
|
|
516
|
-
if (session === this._announcementSession) {
|
|
517
|
-
this._replicationAnnouncementRetryPending = false;
|
|
518
|
-
if (!this.deps.isClosed() &&
|
|
519
|
-
!this.deps.getCloseSignal().aborted &&
|
|
520
|
-
!controller.signal.aborted &&
|
|
521
|
-
this.deps.isAdaptiveReplicating()) {
|
|
522
|
-
void this.deps.callRebalanceParticipationDebounced();
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
else {
|
|
526
|
-
void this.replicationAnnouncementRetryDebounced?.call();
|
|
527
|
-
}
|
|
528
|
-
}
|
|
529
27
|
}
|
|
530
28
|
//# sourceMappingURL=replication-announcement.js.map
|